Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/MGRAST/lib/Auth.pm
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ sub authenticate {
}

# validate the token
my $validation_url = 'https://nexus.api.globusonline.org/goauth/keys/';
my $validation_url = 'https://nexus.api.globusonline.org/goauth/keys/';
my $userdata_url = 'https://nexus.api.globusonline.org/users/';

# token syntax check
my ($user, $sig) = $key =~ /^un=([^\|]+)\|.+SigningSubject=([^\|]+)/;
Expand Down Expand Up @@ -134,6 +135,20 @@ sub authenticate {
# check if a connection exists
my $pref = $master->Preferences->get_objects( { name => 'kbase_user', value => $user } );
if (! scalar(@$pref)) {

# there is no connection, create a fake user and hook them up
$result = `curl -s -X GET "$userdata_url$sig"`;
my $globus_udata;
eval {
$globus_udata = $json->decode($result);
};
if ($@) {
return (undef, "could not reach globus online auth server");
}
#
#
#
#
return (undef, "valid kbase user");
}
}
Expand Down
147 changes: 73 additions & 74 deletions src/MGRAST/lib/JobDB/Job.pm
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ use IO::File;
use Fcntl ':flock';
use MGRAST::Metadata;

use JSON;
use LWP::UserAgent;
use Auth;
use MIME::Base64;

=pod

=head1 NAME
Expand Down Expand Up @@ -909,82 +914,76 @@ sub jobs_mixs_metadata_fast {
#####################

sub fetch_browsepage_in_progress {
my ($self, $user, $count_only) = @_;
my %stage_to_pos = ( 'upload' => 3,
'preprocess' => 4,
'dereplication' => 5,
'screen' => 6,
'genecalling' => 7,
'cluster_aa90' => 8,
'loadAWE' => 9,
'sims' => 10,
'loadDB' => 11,
'done' => 12 );

my $selopt = "( viewable = 0 or viewable is null ) and metagenome_id is not null";
if (ref $user and ( $user->isa("WebServerBackend::User"))) {
unless ($user->has_star_right('edit', 'metagenome')){
$selopt .= ' and metagenome_id in ("';
my @userjobs = $self->get_jobs_for_user_fast($user, 'edit');
unless ( scalar @userjobs > 0 ) {
return [];
}
$selopt .= join('","', map { $_->{'metagenome_id'} } @userjobs).'")';
}
} else {
return [];
}

# get job info
my $statement = "select _id, job_id, name, metagenome_id from Job where ".$selopt;
my $dbh = $self->_master()->db_handle();
my $sth = $dbh->prepare($statement);
$sth->execute;
my $jobdata = $sth->fetchall_arrayref();
$sth->finish;

# get jobs to skip
my $statement = "select job from JobAttributes where tag='deleted' or tag='no_sims_found'";
my $sth = $dbh->prepare($statement);
$sth->execute;
my %skip = map { $_->[0], 1 } @{ $sth->fetchall_arrayref() };
$sth->finish;

if ($count_only) {
my $num = 0;
map { $num += 1 } grep { ! exists $skip{$_->[0]} } @$jobdata;
return $num;
}
return [] unless scalar @$jobdata;

# get job stages
my $data = {};
my $statement = "select job, stage, status from PipelineStage where job in (".join(',', map { $_->[0] } @$jobdata).") order by job";
my $sth = $dbh->prepare($statement);
$sth->execute;
while(my @row = $sth->fetchrow_array()){
next if (exists $skip{$row[0]});
if(exists $data->{$row[0]}){
$data->{$row[0]}->[$stage_to_pos{$row[1]}] = $row[2];
} else {
$data->{$row[0]} = [];
$data->{$row[0]}->[$stage_to_pos{$row[1]}] = $row[2];
}
}
$sth->finish;

foreach my $jobrow (@$jobdata){
next if (exists $skip{$jobrow->[0]});
$data->{$jobrow->[0]}->[0] = $jobrow->[1];
$data->{$jobrow->[0]}->[1] = $jobrow->[2];
$data->{$jobrow->[0]}->[2] = $jobrow->[3];
}
my ($self, $user, $count_only) = @_;

# get jobs
my $jobs = $self->_master->Job->get_objects({ owner => $user});
my $id2job = {};
foreach my $job (@$jobs) {
$id2job->{$job->{job_id}} = $job;
}

my $return_results = [];
foreach my $k (sort keys %$data){
push @$return_results, $data->{$k};
# get mgrast token
my $mgrast_token = undef;
if ($Conf::mgrast_oauth_name && $Conf::mgrast_oauth_pswd) {
my $key = encode_base64($Conf::mgrast_oauth_name.':'.$Conf::mgrast_oauth_pswd);
my $rep = Auth::globus_token($key);
$mgrast_token = $rep ? $rep->{access_token} : undef;
}

# set json handle
my $agent = LWP::UserAgent->new;
my $json = JSON->new;
$json = $json->utf8();
$json->max_size(0);
$json->allow_nonref;

my $stage_titles = { 'upload' => 'Upload',
'preprocess_qc' => 'Sequence Filtering',
'dereplication' => 'Dereplication',
'screen' => 'Sequence Screening',
'genecalling' => 'Gene Calling',
'cluster_aa90' => 'Gene Clustering',
'loadAWE' => 'Calculating Sims',
'sims' => 'Processing Sims',
'loadDB' => 'Loading Database',
'done' => 'Finalizing Data' };

# get awe data
my $stats = {};
eval {
my @args = ('Authorization', "OAuth ".$mgrast_token);
my $get = $agent->get($Conf::awe_url.'/job?query&info.user=mgu'.$user->{_id}, @args);
$stats = $json->decode( $get->content );
};
if ($@ || (! $stats) || (exists($stats->{error}) && $stats->{error})) {
print STDERR "AWE job info retrieval failed for user ".$user->{_id}.": $@".($stats && $stats->{error} ? $stats->{error} : "")."\n";
return $count_only ? 0 : [];
} else {
my $stati = $stats->{data};
if ($count_only) {
return scalar(@$stati);
} else {
my $data_table = [];
foreach my $job (@$stati) {
my $row = { job_id => $job->{info}->{name},
metagenome_id => $id2job->{$job->{info}->{name}}->{metagenome_id},
metagenome_name => $id2job->{$job->{info}->{name}}->{name},
states => [],
status => undef };
my $i=1;
foreach my $stage (@{$job->{tasks}}) {
if (! $row->{status} && ($stage->{remainwork} > 0 || $i == scalar(@{$job->{tasks}}))) {
$row->{status} = ($stage_titles->{$stage->{cmd}->{description}} ? $stage_titles->{$stage->{cmd}->{description}} : $stage->{cmd}->{description}).": ".$stage->{state};
}
push(@{$row->{states}}, { stage => $stage_titles->{$stage->{cmd}->{description}} ? $stage_titles->{$stage->{cmd}->{description}} : $stage->{cmd}->{description}, status => $stage->{state} });
$i++;
}
return $return_results;
push(@$data_table, $row);
}
return $data_table;
}
}
}

sub fetch_browsepage_viewable {
Expand Down
47 changes: 12 additions & 35 deletions src/MGRAST/lib/WebPage/MetagenomeSelect.pm
Original file line number Diff line number Diff line change
Expand Up @@ -437,17 +437,6 @@ sub get_in_progress_table {
my ($self) = @_;
my $user = $self->application->session->user;
my $mgrast = $self->application->data_handle('MGRAST');
my @stages = ('upload', 'preprocess', 'dereplication', 'screen', 'genecalling', 'cluster_aa90', 'loadAWE', 'sims', 'loadDB', 'done');
my %stage_info = ( 'upload' => ['Upload', 3],
'preprocess' => ['Sequence Filtering', 4],
'dereplication' => ['Dereplication', 5],
'screen' => ['Sequence Screening', 6],
'genecalling' => ['Gene Calling', 7],
'cluster_aa90' => ['Gene Clustering', 8],
'loadAWE' => ['Calculating Sims', 9],
'sims' => ['Processing Sims', 10],
'loadDB' => ['Loading Database', 11],
'done' => ['Finalizing Data', 12] );
my $html = "";
my $data_table = $mgrast->Job->fetch_browsepage_in_progress($user);

Expand All @@ -464,25 +453,11 @@ sub get_in_progress_table {
{ name => 'name', filter => 1, visible => 1, sortable => 1 },
{ name => 'progress', width => 135 },
{ name => 'status', filter => 1, sortable => 1 } ]);

my $data = [];
foreach my $row (sort {$b->[0] <=> $a->[0]} @$data_table){
my @in_progress = ();
my $last_stage = '';
foreach my $s (@stages){
if ($s eq 'upload'){
push @in_progress, $self->color_box_for_state('completed', $s);
$last_stage = $s;
} else {
push @in_progress, $self->color_box_for_state($row->[$stage_info{$s}->[1]], $s);
}
$last_stage = $s if $row->[$stage_info{$s}->[1]];
}
if ($last_stage eq 'loadAWE'){
$row->[$stage_info{$last_stage}->[1]] = 'running';
$in_progress[6] = $self->color_box_for_state('running', 'loadAWE');
}
push @$data, [$row->[0], $row->[2], $row->[1], "<div>".join("", @in_progress)."</div>", ($last_stage) ? $stage_info{$last_stage}->[0]." : ".(($row->[$stage_info{$last_stage}->[1]]) ? $row->[$stage_info{$last_stage}->[1]] : "completed") : "" ];
foreach my $row (sort {$b->{job_id} <=> $a->{job_id}} @$data_table){
my @in_progress = map { $self->color_box_for_state($_->{status}, $_->{stage}) } @{$row->{states}};
push @$data, [ $row->{job_id}, $row->{metagenome_id}, $row->{metagenome_name}, "<div>".join("", @in_progress)."</div>", $row->{status} ];
}
$in_progress->data($data);

Expand All @@ -492,15 +467,17 @@ sub get_in_progress_table {

sub color_box_for_state {
my ($self, $state, $stage) = @_;
my %state_to_color = ( 'running' => "#FFBE1E",
'completed' => "#3CA53C",
'unknown' => "#B9B9B9",
'error' => "red" );
my %state_to_color = ( 'completed' => "green",
'in-progress' => "blue",
'queued' => "orange",
'pending' => "gray",
'error' => "red",
'init' => 'gray' );

if ($state and exists $state_to_color{$state}){
return "<div title='".$stage."' style='float:left; height: 14px; width: 12px; margin: 2 0 2 1; background-color:".$state_to_color{$state}.";'></div>";
return "<div title='".$stage."' style='float:left; height: 14px; width: 12px; margin: 2px 0px 2px 1px; background-color:".$state_to_color{$state}.";'></div>";
} else {
return "<div title='".$stage."' style='float:left; height: 14px; width: 12px; margin: 2 0 2 1; background-color:".$state_to_color{'unknown'}.";'></div>";
return "<div title='".$stage."' style='float:left; height: 14px; width: 12px; margin: 2px 0px 2px 1px; background-color:gray;'></div>";
}
}

Expand Down