Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
f22a80d
Merge pull request #5 from teharrison/develop
teharrison Mar 19, 2014
bab2674
Merge pull request #6 from teharrison/develop
teharrison Mar 20, 2014
b447074
Merge pull request #7 from teharrison/develop
teharrison Apr 2, 2014
739cd42
remove project dir from project creation
teharrison Apr 2, 2014
5acd54b
remove refrence of job directory in Job module
teharrison Apr 2, 2014
8fd6b45
removed unused function
teharrison Apr 2, 2014
f0dae93
changed api file download to use shock
teharrison Apr 2, 2014
91c87d1
update download struct
teharrison Apr 2, 2014
ba8f4d3
change inbox view to use shock
teharrison Apr 2, 2014
c3ab2d5
update code because shock uses username and not email for ACLs
teharrison Apr 4, 2014
d0fd484
inbox update: now upload to shock in 2 steps: 1. POST data stream to …
teharrison Apr 4, 2014
cf52470
optimize LWP agent code for shock access
teharrison Apr 4, 2014
d1fcaec
changed file download to stream from shock
teharrison Apr 4, 2014
bebe058
update mgrast token usage and some bug fixes
Apr 4, 2014
de48d3a
fixed return url
Apr 4, 2014
0a165cc
use shock to retrieve metagenome statistics
teharrison Apr 8, 2014
959151d
add missing import
Apr 8, 2014
8489a77
re-organize shock parts
teharrison Apr 8, 2014
6cfa665
Merge branch 'shock' of github.com:teharrison/MG-RAST into shock
teharrison Apr 8, 2014
213173e
changed Analysis.pm to use shock for sims/sequence lookup
teharrison Apr 8, 2014
cebce29
api download of subset index files from shock
teharrison Apr 9, 2014
4b72bfd
fix syntax error
Apr 9, 2014
b0f0f80
fix subset file name
Apr 9, 2014
563ca2e
api annotation download use shock for index retrieval
teharrison Apr 9, 2014
499db37
fix syntax error
Apr 9, 2014
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
411 changes: 192 additions & 219 deletions src/MGRAST/lib/Analysis.pm

Large diffs are not rendered by default.

110 changes: 0 additions & 110 deletions src/MGRAST/lib/JobDB/Job.pm
Original file line number Diff line number Diff line change
Expand Up @@ -272,107 +272,6 @@ sub finish_upload {
}
}

=pod

=item * B<directory> ()

Returns the full path the job directory (without a trailing slash).

=cut

sub directory {
my ($self) = @_;
return $Conf::mgrast_jobs.'/'.$self->job_id;
}

sub dir {
my ($self) = @_;
return $self->directory;
}

=pod

=item * B<download_dir> ()

Returns the full path the download directory inside the job (without a trailing slash).

=cut

sub download_dir {
my ($self, $stage) = @_ ;
if ($stage) {
return $self->directory.'/analysis/';
}
return $self->directory.'/raw/';
}

=pod

=item * B<analysis_dir> ()

Returns the full path the analysis directory inside the job (without a trailing slash).

=cut

sub analysis_dir {
my ($self) = @_;
unless (-d $self->directory.'/analysis') {
chdir($self->directory) or
die("Unable to change directory to ".$self->directory.": $!");
mkdir "analysis", 0777 or
die("Unable to create directory analysis in ".$self->directory.": $!");
}
return $self->directory.'/analysis';
}

=pod

=item * B<download> ()

Returns the name of the project

=cut

sub download {
my ($self , $stage_id , $file) = @_;

if ($file) {
if (open(FH, $self->download_dir($stage_id) . "/" . $file)) {
print "Content-Type:application/x-download\n";
# print "Content-Length: " . length($content) . "\n";
print "Content-Disposition:attachment;filename=".$self->metagenome_id. "." . $file ."\n\n";
#print "<file name='".$self->metagenome_id. "." . $file ."'>";
while (<FH>) {
print $_ ;
}
return ( 1 , "" ) ;
}
else{
return ( 0 , "Could not open download file " . $self->download_dir($stage_id) ."'$file'" );
}
}
elsif (defined $stage_id){
# Download uploaded files
unless ($stage_id){

opendir(DIR , $self->download_dir() ) ;
while (my $file = readdir DIR ){
next unless ($file =~/\.fna|\.fasta|\.sff|\.fastq|\.txt/) ;
print STDERR "Downloading file $file";
$self->download( '' , $file);
}

}

return ( 1 , "" ) ;
}
else{
# return list of download files
}

return 1;
}

=pod

=item * B<get_jobs_for_user> (I<user>, I<right>, I<viewable>)
Expand Down Expand Up @@ -1411,15 +1310,6 @@ sub delete {
$jobgroupjob->delete();
}

# delete the job directory
if (-d $self->directory) {
my $dir = $self->directory;
`rm -rf $dir`;
if (-d $dir) {
die "Could not delete job directory $dir: $@";
}
}

# delete self
$self->SUPER::delete(@_);
return 1;
Expand Down
13 changes: 1 addition & 12 deletions src/MGRAST/lib/JobDB/Project.pm
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,7 @@ sub create_project {
$public = $public ? 1 : 0;

# create project
my $nid = $master->Project->last_id + 1;
my $pdir = $Conf::mgrast_projects;
while (-d "$pdir/$nid") {
$nid++;
}
unless ($pdir && $nid) {
return undef;
}
mkdir("$pdir/$nid", 0777);
mkdir("$pdir/$nid/graphics", 0777);
mkdir("$pdir/$nid/tables", 0777);

my $nid = $master->Project->last_id + 1;
my $attribs = { id => $nid, name => $name, type => 'study', public => $public };
if ($curator && ref($curator)) {
$attribs->{creator} = $curator;
Expand Down
27 changes: 16 additions & 11 deletions src/MGRAST/lib/resources/annotation.pm
Original file line number Diff line number Diff line change
Expand Up @@ -265,19 +265,27 @@ sub prepare_data {
$query .= " ORDER BY seek";
}

my $srcid = $mgdb->_src_id->{$source};
my @head = map { $self->{attributes}{$format}{$_}[1] } sort keys %{$self->{attributes}{$format}};
my $count = 0;
# get shock node for file
my $params = {type => 'metagenome', data_type => 'similarity', stage_name => 'filter.sims', id => 'mgm'.$mgid};
my $sim_node = $self->get_shock_query($params, $self->mgrast_token);
unless ((@$sim_node > 0) && exists($sim_node->[0]{id})) {
$self->return_data({"ERROR" => "Unable to retrieve $format file"}, 500);
}
my $node_id = $sim_node->[0]{id};

open(FILE, "<" . $mgdb->_sim_file($data->{job_id})) || $self->return_data({"ERROR" => "resource database offline"}, 503);
# print html and line headers
my @head = map { $self->{attributes}{$format}{$_}[1] } sort keys %{$self->{attributes}{$format}};
print $cgi->header(-type => 'text/plain', -status => 200, -Access_Control_Allow_Origin => '*');
print join("\t", @head)."\n";


# start query
my $hs = HTML::Strip->new();
my $sth = $mgdb->_dbh->prepare($query);
$sth->execute() or die "Couldn't execute statement: ".$sth->errstr;

# loop through indices and print data
# loop through indexes and print data
my $srcid = $mgdb->_src_id->{$source};
my $count = 0;
while (my @row = $sth->fetchrow_array()) {
my ($md5, $seek, $len, $mmd5) = @row;
my $sql = "";
Expand Down Expand Up @@ -308,10 +316,8 @@ sub prepare_data {
if (@$ann == 0) { next; }
}

# pull data from indexed file
my $rec = '';
seek(FILE, $seek, 0);
read(FILE, $rec, $len);
# pull data from indexed shock file
my $rec = $self->get_shock_file($node_id, undef, $self->mgrast_token, 'seek='.$seek.'&length='.$len);
chomp $rec;
foreach my $line ( split(/\n/, $rec) ) {
my @tabs = split(/\t/, $line);
Expand Down Expand Up @@ -341,7 +347,6 @@ sub prepare_data {
$sth->finish;
$mgdb->_dbh->commit;
print "Download complete. $count rows retrieved\n";
close FILE;
exit 0;
}

Expand Down
159 changes: 32 additions & 127 deletions src/MGRAST/lib/resources/download.pm
Original file line number Diff line number Diff line change
Expand Up @@ -81,157 +81,62 @@ sub instance {

# check id format
my $rest = $self->rest;
my (undef, $mgid) = $rest->[0] =~ /^(mgm)?(\d+\.\d+)$/;
if ((! $mgid) && scalar(@$rest)) {
$self->return_data( {"ERROR" => "invalid id format: ".$rest->[0]}, 400 );
my (undef, $id) = $rest->[0] =~ /^(mgm)?(\d+\.\d+)$/;
if ((! $id) && scalar(@$rest)) {
$self->return_data( {"ERROR" => "invalid id format: " . $rest->[0]}, 400 );
}
# get job

# get database
my $master = $self->connect_to_datasource();
my $job = $master->Job->get_objects( {metagenome_id => $mgid, viewable => 1} );

# get data
my $job = $master->Job->get_objects( {metagenome_id => $id} );
unless ($job && @$job) {
$self->return_data( {"ERROR" => "id $mgid does not exists"}, 404 );
$self->return_data( {"ERROR" => "id $id does not exist"}, 404 );
}
$job = $job->[0];

unless ($job->viewable) {
$self->return_data( {"ERROR" => "id $id is still processing and unavailable"}, 404 );
}

# check rights
unless ($job->{public} || ($self->user && ($self->user->has_right(undef, 'view', 'metagenome', $mgid) || $self->user->has_star_right('view', 'metagenome')))) {
unless ($job->{public} || ($self->user && ($self->user->has_right(undef, 'view', 'metagenome', $id) || $self->user->has_star_right('view', 'metagenome')))) {
$self->return_data( {"ERROR" => "insufficient permissions to view this data"}, 401 );
}

# get data / parameters
my $setlist = $self->setlist($job);
my $stage = $self->cgi->param('stage') || undef;
my $file = $self->cgi->param('file') || undef;
my $stage = $self->cgi->param('stage') || undef;
my $file = $self->cgi->param('file') || undef;
my $setlist = $self->get_download_set($job->{metagenome_id}, $self->mgrast_token);

# return file
# return file from shock
if ($file) {
my ($fdir, $fname);
my $node = undef;
foreach my $set (@$setlist) {
if (($set->{file_id} eq $file) || ($set->{file_name} eq $file)) {
$fname = $set->{file_name};
$fdir = ($set->{stage_id} eq "050") ? $job->download_dir : $job->analysis_dir;
my $subset = exists($self->subset_files->{$set->{stage_name}}) ? 1 : 0;
$self->return_shock_file($set->{node_id}, $set->{file_size}, $set->{file_name}, $self->mgrast_token, $subset);
}
}
if ($fdir && $fname) {
$self->return_file($fdir, $fname);
} else {
$self->return_data( {"ERROR" => "requested file ($file) is not available"}, 404 );
}
$self->return_data( {"ERROR" => "requested file ($file) is not available"}, 404 );
}

# return stage list
# return stage(s) list
my $data = { id => 'mgm'.$job->{metagenome_id},
url => $self->cgi->url."/".$self->name."/mgm".$job->{metagenome_id},
data => [] };
if ($stage) {
my $subsets = [];
$data->{url} .= '?stage='.$stage;
foreach my $set (@$setlist) {
if (($set->{stage_id} eq $stage) || ($set->{stage_name} eq $stage)) {
push @$subsets, $set;
}
}
if (@$subsets > 0) {
$self->return_data($subsets);
} else {
$self->return_data( {"ERROR" => "requested stage ($stage) is not available"}, 404 );
}
} else {
$self->return_data($setlist);
}
}

sub setlist {
my ($self, $job) = @_;

my $rdir = $job->download_dir;
my $adir = $job->analysis_dir;
my $stages = [];

## hardcode upload sequence files: fastq = 050.1, fasta = 050.2
my $fastq = { id => "mgm".$job->metagenome_id,
url => $self->cgi->url.'/'.$self->{name}.'/mgm'.$job->metagenome_id.'?file=050.1',
stage_id => "050",
stage_name => "upload",
file_type => 'fastq',
file_id => "050.1",
file_name => $job->job_id.'.fastq.gz'
};
my $fasta = { id => "mgm".$job->metagenome_id,
url => $self->cgi->url.'/'.$self->{name}.'/mgm'.$job->metagenome_id.'?file=050.2',
stage_id => "050",
stage_name => "upload",
file_type => 'fna',
file_id => "050.2",
file_name => $job->job_id.'.fna.gz'
};

if (opendir(my $dh, $rdir)) {
# add fastq / fastq
my @rawfiles = sort grep { -f "$rdir/$_" } readdir($dh);
if ( any {$_ eq $fastq->{file_name}} @rawfiles ) {
push @$stages, $fastq;
}
if ( any {$_ eq $fasta->{file_name}} @rawfiles ) {
push @$stages, $fasta;
}

closedir $dh;
# start at 3, skip fastq / fasta
my $fnum = 3;
foreach my $rf (@rawfiles) {
next if (($rf eq $fastq->{file_name}) || ($rf eq $fasta->{file_name}));
my $ftype;
if ($rf =~ /\.gz$/) {
($ftype) = $rf =~ /\.([^\.]+)\.gz$/;
} else {
($ftype) = $rf =~ /\.([^\.]+)$/;
}
push(@$stages, { id => "mgm".$job->metagenome_id,
url => $self->cgi->url.'/'.$self->{name}.'/mgm'.$job->metagenome_id.'?file=050.'.$fnum,
stage_id => "050",
stage_name => "upload",
file_type => $ftype,
file_id => "050.".$fnum,
file_name => $rf } );
$fnum += 1;
}
} else {
$self->return_data( {"ERROR" => "job directory could not be opened"}, 404 );
}

if (opendir(my $dh, $adir)) {
my @stagefiles = sort grep { -f "$adir/$_" } readdir($dh);
closedir $dh;
my $stagehash = {};
foreach my $sf (@stagefiles) {
my $ftype;
my ($stageid, $stagename) = $sf =~ /^(\d+)\.([^\.]+)/;
if ($sf =~ /\.gz$/) {
($ftype) = $sf =~ /\.([^\.]+)\.gz$/;
} else {
($ftype) = $sf =~ /\.([^\.]+)$/;
push @{$data->{data}}, $set;
}
next unless ($stageid && $stagename && $ftype);
if (exists($stagehash->{$stageid})) {
$stagehash->{$stageid} += 1;
} else {
$stagehash->{$stageid} = 1;
}
push(@$stages, { id => "mgm".$job->metagenome_id,
url => $self->cgi->url.'/'.$self->{name}.'/mgm'.$job->metagenome_id.'?file='.$stageid.'.'.$stagehash->{$stageid},
stage_id => $stageid,
stage_name => $stagename,
file_type => $ftype,
file_id => $stageid.".".$stagehash->{$stageid},
file_name => $sf } );
}
} else {
$self->return_data( {"ERROR" => "job directory could not be opened"}, 404 );
}

if (@$stages > 0) {
return $stages;
} else {
$self->return_data( {"ERROR" => "no stagefiles found"}, 404 );
# return all
else {
$data->{data} = $setlist;
}
$self->return_data($data);
}

1;
Loading