Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
e43a511
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Jan 26, 2015
3b252a0
Merge branch 'develop' of github.com:MG-RAST/MG-RAST into develop
Feb 4, 2015
1f2d381
Merge branch 'develop' of github.com:teharrison/MG-RAST into develop
Feb 4, 2015
9f22eed
Merge branch 'develop' of github.com:MG-RAST/MG-RAST into develop
Feb 6, 2015
c19d252
Merge branch 'develop' of git@github.com:MG-RAST/MG-RAST into develop
Feb 9, 2015
676ee6c
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 9, 2015
52f9858
Merge branch 'develop' of github.com:teharrison/MG-RAST into develop
Feb 12, 2015
5c555be
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 13, 2015
f879f5a
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 17, 2015
7ed9466
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 17, 2015
76ce335
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 17, 2015
9d7aaf0
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 18, 2015
7c6c7f5
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 18, 2015
6c9d85b
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 18, 2015
50a1563
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 18, 2015
35910e5
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 18, 2015
77b378c
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 18, 2015
237164d
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 19, 2015
6f38a94
Merge branch 'develop' of git@github.com:teharrison/MG-RAST into develop
Feb 19, 2015
8e2b03b
add env to workflows
teharrison Mar 18, 2015
2c56aff
Merge branch 'develop' of github.com:teharrison/MG-RAST into develop
Mar 18, 2015
fdb8ee9
add env to task
Mar 18, 2015
f8419dd
fix env in workflows
Mar 18, 2015
e31c66e
Merge pull request #696 from teharrison/develop
teharrison Mar 18, 2015
3d6e8dd
get error from shock file retrieval
Mar 18, 2015
a399a1f
process error message from shock file retrieval
teharrison Mar 18, 2015
23cef77
fix error handeling in annotation download
teharrison Mar 18, 2015
89680c8
Merge pull request #697 from teharrison/develop
teharrison Mar 18, 2015
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 50 additions & 38 deletions src/MGRAST/lib/resources/annotation.pm
Original file line number Diff line number Diff line change
Expand Up @@ -359,47 +359,59 @@ sub print_batch {
}
}

my $hs = HTML::Strip->new();
for(my $i=0; $i<@{$md5s}; $i++) {
if ($type ne 'md5' && !exists $md5s_to_annot{$md5s->[$i]}) {
next;
}
# pull data from indexed shock file
my $rec = $self->get_shock_file($node_id, undef, $self->mgrast_token, 'seek='.$seeks->[$i].'&length='.$lens->[$i]);
chomp $rec;
foreach my $line (split(/\n/, $rec)) {
my @tabs = split(/\t/, $line);
if ($tabs[0]) {
my @out = ();
my $rid = $hs->parse($tabs[0]);
unless ($mgid && $rid) {
next;
}
$hs->eof;
my $ann = [];
foreach my $key (keys %{$md5s_to_annot{$md5s->[$i]}}) {
push @$ann, $key;
}
if ($type ne 'md5') {
if ($filter && (! $flevel)) {
my @matches = grep {/$filter/} @$ann;
@$ann = @matches;
}
if (@$ann == 0) { next; }
my $hs = HTML::Strip->new();
my $seek_num = scalar(@$seeks);
my $len_num = scalar(@$lens);

for (my $i=0; $i<@{$md5s}; $i++) {
if ( ($type ne 'md5') && (! exists($md5s_to_annot{$md5s->[$i]})) ) {
# missing md5
next;
}
if ( ($seek_num <= $i) || ($len_num <= $i) ) {
# missing seek or length
next;
}
# pull data from indexed shock file
my ($rec, $err) = $self->get_shock_file($node_id, undef, $self->mgrast_token, 'seek='.$seeks->[$i].'&length='.$lens->[$i]);
if ($err) {
# cleanup and exit
$sth->finish;
$mgdb->_dbh->commit;
print "\nERROR downloading: $err\n";
exit 0;
}
chomp $rec;
foreach my $line (split(/\n/, $rec)) {
my @tabs = split(/\t/, $line);
unless ($tabs[0]) { next; }
my @out = ();
my $rid = $hs->parse($tabs[0]);
unless ($rid) { next; }
$hs->eof;
my $ann = [];
foreach my $key (keys %{$md5s_to_annot{$md5s->[$i]}}) {
push @$ann, $key;
}
if ($type ne 'md5') {
if ($filter && (! $flevel)) {
my @matches = grep {/$filter/} @$ann;
@$ann = @matches;
}
if (($format eq 'sequence') && (@tabs == 13)) {
@out = ('mgm'.$mgid."|".$rid, $tabs[1], $tabs[12], join(";", @$ann));
} elsif ($format eq 'similarity') {
@out = ('mgm'.$mgid."|".$rid, @tabs[1..11], join(";", @$ann));
if (@$ann == 0) { next; }
}
if (($format eq 'sequence') && (@tabs == 13)) {
@out = ('mgm'.$mgid."|".$rid, $tabs[1], $tabs[12], join(";", @$ann));
} elsif ($format eq 'similarity') {
@out = ('mgm'.$mgid."|".$rid, @tabs[1..11], join(";", @$ann));
$count += 1;
}
if ($type eq 'md5') {
pop @out;
}
print join("\t", map {$_ || ''} @out)."\n";
$count += 1;
}
if ($type eq 'md5') {
pop @out;
}
print join("\t", map {$_ || ''} @out)."\n";
$count += 1;
}
}
}
return $count;
}
Expand Down
14 changes: 10 additions & 4 deletions src/MGRAST/lib/resources/inbox.pm
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,10 @@ sub demultiplex {
"cmd": {
"args": "-input=\@$fname.$seq_type -input_json=input_attr.json -output_json=output_attr.json -type=$seq_type",
"description": "sequence stats",
"name": "awe_seq_length_stats.pl"
"name": "awe_seq_length_stats.pl",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": ["0"],
"inputs": {
Expand Down Expand Up @@ -627,21 +630,24 @@ sub pair_join {
"cmd": {
"args": "-input=\@$fname -input_json=input_attr.json -output_json=output_attr.json -type=fastq",
"description": "sequence stats",
"name": "awe_seq_length_stats.pl"
"name": "awe_seq_length_stats.pl",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": ["3"],
"inputs": {
"$fname": {
"host": "$shock_url",
"origin": "0",
"origin": "3",
"node": "-",
"attrfile": "input_attr.json"
}
},
"outputs": {
"$fname": {
"host": "$shock_url",
"origin": "0",
"origin": "3",
"node": "-",
"type": "update",
"attrfile": "output_attr.json"
Expand Down
6 changes: 3 additions & 3 deletions src/MGRAST/lib/resources/metadata.pm
Original file line number Diff line number Diff line change
Expand Up @@ -474,9 +474,9 @@ sub process_file {
my $nid = $self->cgi->param('node_id');
$node = $self->get_shock_node($nid, $self->mgrast_token);
$fname = $node->{file}{name};
my $res = $self->get_shock_file($nid, "$tmp_dir/$fname", $self->mgrast_token);
unless ($res) {
$self->return_data({"ERROR" => "Unable to retrieve file from Shock server"}, 500);
my ($res, $err) = $self->get_shock_file($nid, "$tmp_dir/$fname", $self->mgrast_token);
if ($err) {
$self->return_data({"ERROR" => $err}, 500);
}
}
# bad POST
Expand Down
5 changes: 4 additions & 1 deletion src/MGRAST/lib/resources/metagenome.pm
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,10 @@ sub prepare_data {
sub clean_stats {
my ($self, $stat_node_id) = @_;

my $stats = $self->json->decode($self->get_shock_file($stat_node_id, undef, $self->mgrast_token));
my ($stats, $err) = $self->json->decode($self->get_shock_file($stat_node_id, undef, $self->mgrast_token));
if ($err) {
$self->return_data( {"ERROR" => $err}, 500 );
}
# seq stats
foreach my $key (keys %{$stats->{sequence_stats}}) {
$stats->{sequence_stats}{$key} = $self->toFloat($stats->{sequence_stats}{$key});
Expand Down
12 changes: 10 additions & 2 deletions src/MGRAST/lib/resources/notebook.pm
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,11 @@ sub prepare_data {
$obj->{permission} = $node->{attributes}{permission} || 'edit';
$obj->{description} = $node->{attributes}{description} || '';
if ($self->cgi->param('verbosity') && ($self->cgi->param('verbosity') eq 'full')) {
$obj->{notebook} = $self->json->decode( encode_utf8($self->get_shock_file($obj->{id}, undef, $self->shock_auth())) );
my ($content, $err) = $self->get_shock_file($obj->{id}, undef, $self->shock_auth());
if ($err) {
$self->return_data( {"ERROR" => $err}, 500 );
}
$obj->{notebook} = $self->json->decode( encode_utf8($content) );
}
push @$objects, $obj;
}
Expand All @@ -277,7 +281,11 @@ sub prepare_data {
sub clone_notebook {
my ($self, $node, $params) = @_;

my $file = $self->json->decode( encode_utf8($self->get_shock_file($node->{id}, undef, $self->shock_auth())) );
my ($file, $err) = $self->get_shock_file($node->{id}, undef, $self->shock_auth());
if ($err) {
$self->return_data( {"ERROR" => $err}, 500 );
}
$file = $self->json->decode( encode_utf8($file) );
my $attr = { name => $node->{attributes}{name}.'_copy',
nbid => undef,
type => $node->{attributes}{type} ? $node->{attributes}{type} : 'generic',
Expand Down
10 changes: 7 additions & 3 deletions src/MGRAST/lib/resources/resource.pm
Original file line number Diff line number Diff line change
Expand Up @@ -907,6 +907,7 @@ sub get_shock_preauth {
}

# write file content to given filepath, else return file content as string
# returns tuple: (content, error_msg)
sub get_shock_file {
my ($self, $id, $file, $auth, $index, $authPrefix) = @_;

Expand All @@ -926,13 +927,16 @@ sub get_shock_file {
my $url = $Conf::shock_url.'/node/'.$id.'?download'.($index ? '&'.$index : '');
$response = $self->agent->get($url, @args);
};

if ($@ || (! $response)) {
return undef;
return (undef, "Unable to conect to Shock server");
} elsif ($response->is_error) {
return (undef, $response->code.": ".$response->message);
} elsif ($file) {
close($fhdl);
return 1;
return (1, undef);
} else {
return $response->content;
return ($response->content, undef);
}
}

Expand Down
20 changes: 16 additions & 4 deletions src/MGRAST/lib/resources/validation.pm
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,10 @@ sub template {
}

# get the shock template file
my $template_str = $self->get_shock_file($id, undef, $self->{token});
my ($template_str, $err) = $self->get_shock_file($id, undef, $self->{token});
if ($err) {
$self->return_data( {"ERROR" => $err}, 500 );
}
my $json = JSON->new->allow_nonref;
my $template = $json->decode($template_str);

Expand Down Expand Up @@ -347,7 +350,10 @@ sub data {
$template_attributes = $template_node->{attributes};

# getting file
my $template_str = $self->get_shock_file($template_id, undef, $self->{token});
my ($template_str, $err) = $self->get_shock_file($template_id, undef, $self->{token});
if ($err) {
$self->return_data( {"ERROR" => $err}, 500 );
}
$template = $json->decode($template_str);

# unless ($template_attributes->{data_type} eq 'template') {
Expand All @@ -359,12 +365,18 @@ sub data {
$template_attributes = $template_node->{attributes};

# getting file
my $template_str = $self->get_shock_file($Conf::mgrast_md_template_node_id, undef, $self->mgrast_token);
my ($template_str, $err) = $self->get_shock_file($Conf::mgrast_md_template_node_id, undef, $self->mgrast_token);
if ($err) {
$self->return_data( {"ERROR" => $err}, 500 );
}
$template = $json->decode($template_str);
}
# check shock type to be template

my $data_str = $self->get_shock_file($data_id, undef, $self->{token});
my ($data_str, $err) = $self->get_shock_file($data_id, undef, $self->{token});
if ($err) {
$self->return_data( {"ERROR" => $err}, 500 );
}
my $data = $json->decode($data_str);

my $data_status = { "valid" => 1,
Expand Down
5 changes: 4 additions & 1 deletion src/MGRAST/workflows/demultiplex.awf
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@
"cmd": {
"args": "-f [% file_type %] -b @[% bar_file %] -i @[% seq_file %]",
"description": "demultiplex",
"name": "demultiplex.py"
"name": "demultiplex.py",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": [],
"inputs": {
Expand Down
10 changes: 8 additions & 2 deletions src/MGRAST/workflows/pair_join.awf
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@
"cmd": {
"args": "[% retain %]-m 8 -p 10 -t . -o [% out_file %].fastq @[% p1_file %] @[% p2_file %]",
"description": "merge mate-pairs",
"name": "pairend_join.py"
"name": "pairend_join.py",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": [],
"inputs": {
Expand Down Expand Up @@ -47,7 +50,10 @@
"cmd": {
"args": "-input=@[% out_file %].fastq -input_json=input_attr.json -output_json=output_attr.json -type=fastq",
"description": "sequence stats",
"name": "awe_seq_length_stats.pl"
"name": "awe_seq_length_stats.pl",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": ["0"],
"inputs": {
Expand Down
20 changes: 16 additions & 4 deletions src/MGRAST/workflows/pair_join_demultiplex.awf
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@
"cmd": {
"args": "[% retain %]-m 8 -p 10 -t . -r -i [% index_file %] -o [% out_file %].join.fastq @[% p1_file %] @[% p2_file %]",
"description": "merge mate-pairs",
"name": "pairend_join.py"
"name": "pairend_join.py",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": [],
"inputs": {
Expand Down Expand Up @@ -51,7 +54,10 @@
"cmd": {
"args": "-r -i @[% index_file %] -p [% out_file %] -o [% out_file %].barcodes.txt",
"description": "build barcodes",
"name": "index2barcode.py"
"name": "index2barcode.py",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": [],
"inputs": {
Expand All @@ -77,7 +83,10 @@
"cmd": {
"args": "-input=@[% out_file %].join.fastq -input_json=input_attr.json -output_json=output_attr.json -type=fastq",
"description": "sequence stats",
"name": "awe_seq_length_stats.pl"
"name": "awe_seq_length_stats.pl",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": ["0"],
"inputs": {
Expand All @@ -104,7 +113,10 @@
"cmd": {
"args": "-f fastq -b @[% out_file %].barcodes.txt -i @[% out_file %].join.fastq",
"description": "demultiplex",
"name": "demultiplex.py"
"name": "demultiplex.py",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": ["1", "2"],
"inputs": {
Expand Down
5 changes: 4 additions & 1 deletion src/MGRAST/workflows/seq_stats.awf
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@
"cmd": {
"args": "-input=@[% seq_file %] -input_json=input_attr.json -output_json=output_attr.json -type=[% file_type %]",
"description": "sequence stats",
"name": "awe_seq_length_stats.pl"
"name": "awe_seq_length_stats.pl",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": [],
"inputs": {
Expand Down
10 changes: 8 additions & 2 deletions src/MGRAST/workflows/sff_to_fastq.awf
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@
"cmd": {
"args": "-Q @[% sff_file %] -s [% fastq_file %]",
"description": "sff to fastq",
"name": "sff_extract_0_2_8"
"name": "sff_extract_0_2_8",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": [],
"inputs": {
Expand All @@ -42,7 +45,10 @@
"cmd": {
"args": "-input=@[% fastq_file %] -input_json=input_attr.json -output_json=output_attr.json -type=fastq",
"description": "sequence stats",
"name": "awe_seq_length_stats.pl"
"name": "awe_seq_length_stats.pl",
"environ" : {
"public" : { "PERL5LIB" : "/root/pipeline/lib:/root/pipeline/conf" }
}
},
"dependsOn": ["0"],
"inputs": {
Expand Down