Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
48c8581
add awe function
teharrison Feb 17, 2015
f6174ce
add inbox awe job info to nodes, calculate file_info and refresh job …
teharrison Feb 17, 2015
2d33195
resolve conflict
teharrison Feb 17, 2015
db33bbc
correct variable name
teharrison Feb 17, 2015
972062b
added pipeline job info to input file node attributes
teharrison Feb 17, 2015
7d348fe
set type for excel files
teharrison Feb 18, 2015
1c3b454
set type for excel files
teharrison Feb 18, 2015
b15674e
add delete function
teharrison Feb 18, 2015
75a6f74
status message updates
teharrison Feb 18, 2015
e1b9f05
accept delete method
teharrison Feb 18, 2015
e9de7bf
fix filename
teharrison Feb 18, 2015
d1b731b
fix status message
teharrison Feb 18, 2015
8d9f3e4
check if stats has been ran previously
teharrison Feb 19, 2015
74cf660
update shock node after validation
teharrison Feb 19, 2015
b69b946
update output filenames
teharrison Feb 19, 2015
784c6fe
add rename and single file view, update workflows
teharrison Feb 19, 2015
7b7e1fb
typo fixes
teharrison Feb 19, 2015
2d4a6d2
fix sff2fastq output filename
teharrison Feb 20, 2015
f54477b
add seq stats calc to output of sff2fastq workflow
teharrison Feb 20, 2015
69a5548
add seq stats calc to output of pair_join workflow
teharrison Feb 20, 2015
a5de3ec
attribute update
teharrison Feb 23, 2015
124a50c
add seq stats calc to output of workflows
teharrison Feb 23, 2015
d2c7afc
add seq stats calc to output of workflows
teharrison Feb 23, 2015
6671d4f
update job creation defaults
teharrison Feb 23, 2015
868995d
resolve conflicts, add pending to request list
teharrison Feb 23, 2015
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/MGRAST/cgi/api.cgi
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ if (lc($request_method) eq 'options') {
-status => 200,
-type => 'text/plain',
-charset => 'UTF-8',
-Access_Control_Allow_Methods => 'POST, GET, OPTIONS, PUT',
-Access_Control_Allow_Methods => 'POST, GET, OPTIONS, PUT, DELETE',
-Access_Control_Allow_Headers => 'AUTH, AUTHORIZATION'
);
print "";
Expand Down
507 changes: 378 additions & 129 deletions src/MGRAST/lib/resources/inbox.pm

Large diffs are not rendered by default.

27 changes: 27 additions & 0 deletions src/MGRAST/lib/resources/job.pm
Original file line number Diff line number Diff line change
Expand Up @@ -303,12 +303,24 @@ sub job_action {
$self->return_data( {"ERROR" => "unable to obtain sequence file info from shock node ".$nodeid}, 500 );
}
}
# fix assembly defaults
if (exists($post->{sequencing_method_guess}) && ($post->{sequencing_method_guess} eq "assembled")) {
$post->{assembled} = 'yes';
$post->{filter_ln} = 'no';
$post->{filter_ambig} = 'no';
$post->{dynamic_trim} = 'no';
$post->{dereplicate} = 'no';
$post->{bowtie} = 'no';
}
# set pipeline defaults if missing
foreach my $key (@{$self->pipeline_opts}) {
if (exists($self->pipeline_defaults->{$key}) && (! exists($post->{$key}))) {
$post->{$key} = $self->pipeline_defaults->{$key};
}
}
if ($post->{file_type} eq 'fasta') {
$post->{file_type} = 'fna';
}
# fix booleans
foreach my $key (keys %$post) {
if ($post->{$key} eq 'yes') {
Expand Down Expand Up @@ -355,6 +367,21 @@ sub job_action {
awe_id => $aid,
log => join("\n", @log)
};
# update inbox attributes
my $node = $self->get_shock_node($post->{input_id}, $self->mgrast_token);
my $attr = $node->{attributes};
my $action = {
id => $aid,
name => "pipeline",
status => "queued",
start => strftime("%Y-%m-%dT%H:%M:%S", gmtime)
};
if (exists $attr->{actions}) {
push @{$attr->{actions}}, $action;
} else {
$attr->{actions} = [$action];
}
$self->update_shock_node($post->{input_id}, $attr, $self->mgrast_token);
} else {
$self->return_data( {"ERROR" => "Unknown error, missing AWE job ID:\n".join("\n", @log)}, 500 );
}
Expand Down
15 changes: 11 additions & 4 deletions src/MGRAST/lib/resources/metadata.pm
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,7 @@ sub process_file {
# get metadata file
my $tmp_dir = $Conf::temp;
my $fname = "";
my $node = {};

# uploaded
if ($self->cgi->param('upload')) {
Expand Down Expand Up @@ -470,10 +471,10 @@ sub process_file {
}
# from shock node
elsif ($self->cgi->param('node_id')) {
my $nid = $self->cgi->param('node_id');
my $node = $self->get_shock_node($nid, $self->mgrast_token);
$fname = $node->{file}{name};
my $res = $self->get_shock_file($nid, "$tmp_dir/$fname", $self->mgrast_token);
my $nid = $self->cgi->param('node_id');
$node = $self->get_shock_node($nid, $self->mgrast_token);
$fname = $node->{file}{name};
my $res = $self->get_shock_file($nid, "$tmp_dir/$fname", $self->mgrast_token);
unless ($res) {
$self->return_data({"ERROR" => "Unable to retrieve file from Shock server"}, 500);
}
Expand All @@ -489,6 +490,12 @@ sub process_file {
# run different actions
if ($type eq 'validate') {
if ($is_valid) {
if ($node && ref($node)) {
# validate shock node
my $attr = $node->{attributes};
$attr->{data_type} = 'metadata';
$self->update_shock_node($node->{id}, $attr, $self->mgrast_token);
}
delete $md_obj->{is_valid};
$data = {is_valid => 1, message => undef, metadata => $md_obj};
} else {
Expand Down
46 changes: 46 additions & 0 deletions src/MGRAST/lib/resources/resource.pm
Original file line number Diff line number Diff line change
Expand Up @@ -860,6 +860,29 @@ sub get_shock_node {
}
}

# delete node
sub delete_shock_node {
my ($self, $id, $auth, $authPrefix) = @_;

if (! $authPrefix) {
$authPrefix = "OAuth";
}

my $response = undef;
eval {
my @args = $auth ? ('Authorization', "$authPrefix $auth") : ();
my $get = $self->agent->delete($Conf::shock_url.'/node/'.$id, @args);
$response = $self->json->decode( $get->content );
};
if ($@ || (! ref($response))) {
return undef;
} elsif (exists($response->{error}) && $response->{error}) {
$self->return_data( {"ERROR" => "Unable to DELETE node $id from Shock: ".$response->{error}[0]}, $response->{status} );
} else {
return $response->{data};
}
}

# get the shock preauth url for a file
sub get_shock_preauth {
my ($self, $id, $auth, $fn, $authPrefix) = @_;
Expand Down Expand Up @@ -1000,6 +1023,29 @@ sub awe_job_action {
}
}

# get job document
sub get_awe_job {
my ($self, $id, $auth, $authPrefix) = @_;

if (! $authPrefix) {
$authPrefix = "OAuth";
}

my $response = undef;
eval {
my @args = $auth ? ('Authorization', "$authPrefix $auth") : ();
my $get = $self->agent->get($Conf::awe_url.'/job/'.$id, @args);
$response = $self->json->decode( $get->content );
};
if ($@ || (! ref($response))) {
return undef;
} elsif (exists($response->{error}) && $response->{error}) {
$self->return_data( {"ERROR" => "Unable to GET job $id from AWE: ".$response->{error}[0]}, $response->{status} );
} else {
return $response->{data};
}
}

# get list of jobs for query
sub get_awe_query {
my ($self, $params, $auth, $authPrefix) = @_;
Expand Down
9 changes: 4 additions & 5 deletions src/MGRAST/workflows/demultiplex.awf
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,11 @@
"outputs": {[% outputs %]
},
"userattr": {
"file_format": "[% file_type %]",
"seq_format": "bp",
"parent_file": "[% seq_file_id %]"
"parent_seq_file": "[% seq_file_id %]"
"parent_barcode_file": "[% bar_file_id %]"
},
"taskid": "0",
"totalwork": 1
}
}[% seq_stats %]
]
}
}
39 changes: 32 additions & 7 deletions src/MGRAST/workflows/pair_join.awf
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"tasks": [
{
"cmd": {
"args": "[% retain %]-m 8 -p 10 -t . -o [% out_file %] @[% p1_file %] @[% p2_file %]",
"args": "[% retain %]-m 8 -p 10 -t . -o [% out_file %].fastq @[% p1_file %] @[% p2_file %]",
"description": "merge mate-pairs",
"name": "pairend_join.py"
},
Expand All @@ -30,20 +30,45 @@
}
},
"outputs": {
"[% out_file %]": {
"[% out_file %].fastq": {
"host": "[% shock_url %]",
"node": "-",
"attrfile": "userattr.json"
}
},
"userattr": {
"file_format": "fastq",
"seq_format": "bp",
"parent_file_1": "[% p1_file_id %]",
"parent_file_2": "[% p2_file_id %]"
"parent_seq_file_1": "[% p1_file_id %]",
"parent_seq_file_2": "[% p2_file_id %]"
},
"taskid": "0",
"totalwork": 1
},
{
"cmd": {
"args": "-input=@[% out_file %].fastq -input_json=input_attr.json -output_json=output_attr.json -type=fastq",
"description": "sequence stats",
"name": "awe_seq_length_stats.pl"
},
"dependsOn": ["0"],
"inputs": {
"[% out_file %].fastq": {
"host": "[% shock_url %]",
"origin": "0",
"node": "-",
"attrfile": "input_attr.json"
}
},
"outputs": {
"[% out_file %].fastq": {
"host": "[% shock_url %]",
"origin": "0",
"node": "-",
"type": "update",
"attrfile": "output_attr.json"
}
},
"taskid": "1",
"totalwork": 1
}
]
}
}
67 changes: 45 additions & 22 deletions src/MGRAST/workflows/pair_join_demultiplex.awf
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"tasks": [
{
"cmd": {
"args": "[% retain %]-m 8 -p 10 -t . -r -i [% index_file %] -o [% prefix %].join.fastq @[% p1_file %] @[% p2_file %]",
"args": "[% retain %]-m 8 -p 10 -t . -r -i [% index_file %] -o [% out_file %].join.fastq @[% p1_file %] @[% p2_file %]",
"description": "merge mate-pairs",
"name": "pairend_join.py"
},
Expand All @@ -34,24 +34,22 @@
}
},
"outputs": {
"[% prefix %].join.fastq": {
"[% out_file %].join.fastq": {
"host": "[% shock_url %]",
"node": "-",
"attrfile": "userattr.json"
}
},
"userattr": {
"file_format": "fastq",
"seq_format": "bp",
"parent_file_1": "[% p1_file_id %]",
"parent_file_2": "[% p2_file_id %]"
"parent_seq_file_1": "[% p1_file_id %]",
"parent_seq_file_2": "[% p2_file_id %]"
},
"taskid": "0",
"totalwork": 1
},
{
"cmd": {
"args": "-r -i @[% index_file %] -p [% prefix %] -o [% prefix %].barcodes.txt",
"args": "-r -i @[% index_file %] -p [% out_file %] -o [% out_file %].barcodes.txt",
"description": "build barcodes",
"name": "index2barcode.py"
},
Expand All @@ -63,33 +61,59 @@
}
},
"outputs": {
"[% prefix %].barcodes.txt": {
"[% out_file %].barcodes.txt": {
"host": "[% shock_url %]",
"node": "-",
"attrfile": "userattr.json"
}
},
"userattr": {
"file_format": "text",
"parent_file": "[% index_id %]"
"parent_index_file": "[% index_id %]"
},
"taskid": "1",
"totalwork": 1
},
{
"cmd": {
"args": "-f fastq -b @[% prefix %].barcodes.txt -i @[% prefix %].join.fastq",
"args": "-input=@[% out_file %].join.fastq -input_json=input_attr.json -output_json=output_attr.json -type=fastq",
"description": "sequence stats",
"name": "awe_seq_length_stats.pl"
},
"dependsOn": ["0"],
"inputs": {
"[% out_file %].join.fastq": {
"host": "[% shock_url %]",
"origin": "0",
"node": "-",
"attrfile": "input_attr.json"
}
},
"outputs": {
"[% out_file %].join.fastq": {
"host": "[% shock_url %]",
"origin": "0",
"node": "-",
"type": "update",
"attrfile": "output_attr.json"
}
},
"taskid": "2",
"totalwork": 1
}
{
"cmd": {
"args": "-f fastq -b @[% out_file %].barcodes.txt -i @[% out_file %].join.fastq",
"description": "demultiplex",
"name": "demultiplex.py"
},
"dependsOn": ["0", "1"],
"dependsOn": ["1", "2"],
"inputs": {
"[% prefix %].join.fastq": {
"[% out_file %].join.fastq": {
"host": "[% shock_url %]",
"origin": "0",
"origin": "2",
"node": "-"
},
"[% prefix %].barcodes.txt": {
"[% out_file %].barcodes.txt": {
"host": "[% shock_url %]",
"origin": "1",
"node": "-"
Expand All @@ -98,13 +122,12 @@
"outputs": {[% outputs %]
},
"userattr": {
"file_format": "fastq",
"seq_format": "bp",
"parent_file_1": "[% p1_file_id %]",
"parent_file_2": "[% p2_file_id %]"
"parent_seq_file_1": "[% p1_file_id %]",
"parent_seq_file_2": "[% p2_file_id %]",
"parent_index_file": "[% index_id %]"
},
"taskid": "0",
"taskid": "3",
"totalwork": 1
}
}[% seq_stats %]
]
}
}
Loading