Skip to content
Merged

Shock #524

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 74 additions & 39 deletions src/MGRAST/lib/Analysis.pm
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ no warnings('once');

use Auth;
use Conf;
use Babel::lib::Babel;

use List::Util qw(max min sum first);
use List::MoreUtils qw(natatime);
Expand All @@ -23,14 +22,23 @@ use File::Temp qw/ tempfile tempdir /;
sub new {
my ($class, $job_dbh, $dbh) = @_;

# get ach object
my $ach = new Babel::lib::Babel;
# get ach object if have lib
my $ach = undef;
eval {
require Babel::lib::Babel;
Babel::lib::Babel->import();
$ach = new Babel::lib::Babel;
};

# get memcache object
my $memd = new Cache::Memcached {'servers' => [$Conf::web_memcache || "kursk-2.mcs.anl.gov:11211"], 'debug' => 0, 'compress_threshold' => 10_000};
my $memd = undef;
eval {
require Cache::Memcached;
Cache::Memcached->import();
$memd = new Cache::Memcached {'servers' => [$Conf::web_memcache || ''], 'debug' => 0, 'compress_threshold' => 10_000};
};

# connect to database

unless ($dbh) {
eval {
my $dbms = $Conf::mgrast_dbms;
Expand Down Expand Up @@ -318,6 +326,9 @@ sub _get_where_str {
sub _run_fraggenescan {
my ($self, $fasta) = @_;

unless ($Conf::run_fraggenescan) {
return "";
}
my ($infile_hdl, $infile_name) = tempfile("fgs_in_XXXXXXX", DIR => $Conf::temp, SUFFIX => '.fna');
print $infile_hdl $fasta;
close $infile_hdl;
Expand Down Expand Up @@ -699,7 +710,7 @@ sub _search_annotations {
my $data = {};
my $jobs = [];
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $cdata = $self->_memd->get($mg.$cache_key);
my $cdata = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
if ($cdata) { $data->{$mg} = $cdata; }
else { push @$jobs, $j; }
}
Expand All @@ -714,8 +725,10 @@ sub _search_annotations {
}
$sth->finish;

foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
if ($self->_memd) {
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
}
}
$self->_dbh->commit;
return $data;
Expand Down Expand Up @@ -1109,10 +1122,16 @@ sub get_organism_abundance_for_source {
sub get_organisms_with_contig_for_source {
my ($self, $src, $num, $len) = @_;

my $job_orgs = $self->get_organism_abundance_for_source($src);
my @job_ctgs = map { [$_->[0], $_->[1], $job_orgs->{$_->[0]}] } grep { exists $job_orgs->{$_->[0]} } @{ $self->ach->get_organism_with_contig_list($num, $len) };
# [ org_id, org_name, abundance ]
return \@job_ctgs;
if ($self->ach) {
my $job_orgs = $self->get_organism_abundance_for_source($src);
my @job_ctgs = map { [$_->[0], $_->[1], $job_orgs->{$_->[0]}] }
grep { exists $job_orgs->{$_->[0]} }
@{ $self->ach->get_organism_with_contig_list($num, $len) };
# [ org_id, org_name, abundance ]
return \@job_ctgs;
} else {
return [];
}
}

sub get_md5_evals_for_organism_source {
Expand Down Expand Up @@ -1204,7 +1223,7 @@ sub get_rarefaction_curve {
# calculate alpha diversity
if ($get_alpha) {
foreach my $mg (keys %$raw_data) {
my $cdata = $self->_memd->get($mg.$cache_key."alpha");
my $cdata = $self->_memd ? $self->_memd->get($mg.$cache_key."alpha") : undef;
unless ($cdata) {
my $h1 = 0;
my $sum = sum values %{$raw_data->{$mg}};
Expand All @@ -1218,7 +1237,9 @@ sub get_rarefaction_curve {
if ($p > 0) { $h1 += ($p * log(1/$p)) / log(2); }
}
$mg_alpha->{$mg} = 2 ** $h1;
$self->_memd->set($mg.$cache_key."alpha", $mg_alpha->{$mg}, $self->_expire);
if ($self->_memd) {
$self->_memd->set($mg.$cache_key."alpha", $mg_alpha->{$mg}, $self->_expire);
}
} else {
$mg_alpha->{$mg} = $cdata;
}
Expand All @@ -1229,7 +1250,7 @@ sub get_rarefaction_curve {

# calculate rarefaction (x, y)
foreach my $mg (keys %$raw_data) {
my $cdata = $self->_memd->get($mg.$cache_key."curve");
my $cdata = $self->_memd ? $self->_memd->get($mg.$cache_key."curve") : undef;
unless ($cdata) {
my @nums = sort {$a <=> $b} values %{$raw_data->{$mg}};
my $k = scalar @nums;
Expand All @@ -1246,7 +1267,9 @@ sub get_rarefaction_curve {
map { $curr += exp( $self->_nCr2ln($nseq - $_, $n) - $coeff ) } @nums;
push @{ $mg_rare->{$mg} }, [ $n, $k - $curr ];
}
$self->_memd->set($mg.$cache_key."curve", $mg_rare->{$mg}, $self->_expire);
if ($self->_memd) {
$self->_memd->set($mg.$cache_key."curve", $mg_rare->{$mg}, $self->_expire);
}
} else {
$mg_rare->{$mg} = $cdata;
}
Expand Down Expand Up @@ -1285,7 +1308,7 @@ sub _get_abundance_for_hierarchy {
$jobs = $self->_jobs;
} else {
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $cdata = $self->_memd->get($mg.$cache_key);
my $cdata = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
if ($cdata) { push @$data, @$cdata; }
else { push @$jobs, $j; }
}
Expand Down Expand Up @@ -1329,7 +1352,9 @@ sub _get_abundance_for_hierarchy {
push @$data, [ $self->_mg_map->{$curr}, $h, $num ];
push @$cdata, [ $self->_mg_map->{$curr}, $h, $num ];
}
$self->_memd->set($self->_mg_map->{$curr}.$cache_key, $cdata, $self->_expire);
if ($self->_memd) {
$self->_memd->set($self->_mg_map->{$curr}.$cache_key, $cdata, $self->_expire);
}
# reset
$hier = {};
$curr = $job;
Expand Down Expand Up @@ -1361,7 +1386,9 @@ sub _get_abundance_for_hierarchy {
push @$data, [ $self->_mg_map->{$job}, $h, $num ];
push @$cdata, [ $self->_mg_map->{$job}, $h, $num ];
}
$self->_memd->set($self->_mg_map->{$job}.$cache_key, $cdata, $self->_expire);
if ($self->_memd) {
$self->_memd->set($self->_mg_map->{$job}.$cache_key, $cdata, $self->_expire);
}
}
$self->_dbh->commit;
return $data;
Expand Down Expand Up @@ -1549,8 +1576,8 @@ sub get_organisms_for_md5s {
$jobs = $self->_jobs;
} else {
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $c = $self->_memd->get($mg.$cache_key);
my $m = $self->_memd->get($mg.$cache_key."md5s");
my $c = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
my $m = $self->_memd ? $self->_memd->get($mg.$cache_key."md5s") : undef;
if ($c && $m) {
$data->{$mg} = $c;
$mdata{$mg} = $m;
Expand Down Expand Up @@ -1595,7 +1622,7 @@ sub get_organisms_for_md5s {
map { $mdata{$mg}{$_} = $mg_md5_abund->{$mg}{$_} } grep { exists $mg_md5_abund->{$mg}{$_} } @{$row[17]};
}
$sth->finish;
unless ($qmd5s) {
if ((! $qmd5s) && $self->_memd) {
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
$self->_memd->set($mg.$cache_key."md5s", $mdata{$mg}, $self->_expire);
Expand Down Expand Up @@ -1638,8 +1665,8 @@ sub get_ontology_for_md5s {
$jobs = $self->_jobs;
} else {
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $c = $self->_memd->get($mg.$cache_key);
my $m = $self->_memd->get($mg.$cache_key."md5s");
my $c = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
my $m = $self->_memd ? $self->_memd->get($mg.$cache_key."md5s") : undef;
if ($c && $m) {
$data->{$mg} = $c;
$mdata{$mg} = $m;
Expand Down Expand Up @@ -1678,7 +1705,7 @@ sub get_ontology_for_md5s {
map { $mdata{$mg}{$_} = $mg_md5_abund->{$mg}{$_} } grep { exists $mg_md5_abund->{$mg}{$_} } @{$row[10]};
}
$sth->finish;
unless ($qmd5s) {
if ((! $qmd5s) && $self->_memd) {
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
$self->_memd->set($mg.$cache_key."md5s", $mdata{$mg}, $self->_expire);
Expand Down Expand Up @@ -1718,7 +1745,7 @@ sub get_functions_for_md5s {
$jobs = $self->_jobs;
} else {
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $c = $self->_memd->get($mg.$cache_key);
my $c = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
if ($c) { $data->{$mg} = $c; }
else { push @$jobs, $j; }
}
Expand Down Expand Up @@ -1752,7 +1779,7 @@ sub get_functions_for_md5s {
push @{$data->{$mg}}, [ $mg, $self->_id_src->{$row[1]}, @row[2,3], $sub_abund, @row[4..9], join(";", @{$row[10]}) ];
}
$sth->finish;
unless ($qmd5s) {
if ((! $qmd5s) && $self->_memd) {
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
}
Expand All @@ -1773,7 +1800,7 @@ sub get_lca_data {
my $data = {};
my $jobs = [];
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $c = $self->_memd->get($mg.$cache_key);
my $c = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
if ($c) { $data->{$mg} = $c; }
else { push @$jobs, $j; }
}
Expand All @@ -1797,8 +1824,10 @@ sub get_lca_data {
push @{$data->{$mg}}, [ $mg, @tax, @row[2..8] ];
}
$sth->finish;
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
if ($self->_memd) {
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
}
}
$self->_dbh->commit;
return [ map { @$_ } values %$data ];
Expand All @@ -1820,7 +1849,7 @@ sub get_md5_data {
$jobs = $self->_jobs;
} else {
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $c = $self->_memd->get($mg.$cache_key);
my $c = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
if ($c) { $data->{$mg} = $c; }
else { push @$jobs, $j; }
}
Expand Down Expand Up @@ -1848,8 +1877,10 @@ sub get_md5_data {
push @{ $data->{$mg} }, [ $mg, @row ];
}
$sth->finish;
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
if ($self->_memd) {
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
}
}
$self->_dbh->commit;
return [ map { @$_ } values %$data ];
Expand All @@ -1870,7 +1901,7 @@ sub get_md5_abundance {
$jobs = $self->_jobs;
} else {
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $c = $self->_memd->get($mg.$cache_key);
my $c = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
if ($c) { $data->{$mg} = $c; }
else { push @$jobs, $j; }
}
Expand Down Expand Up @@ -1905,8 +1936,10 @@ sub get_md5_abundance {
}
$sth->finish;
}
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
if ($self->_memd) {
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
}
}
$self->_dbh->commit;
return $data;
Expand Down Expand Up @@ -1942,7 +1975,7 @@ sub _get_annotation_md5 {
my $data = {};
my $jobs = [];
while ( my ($mg, $j) = each %{$self->_job_map} ) {
my $c = $self->_memd->get($mg.$cache_key);
my $c = $self->_memd ? $self->_memd->get($mg.$cache_key) : undef;
if ($c) { $data->{$mg} = $c; }
else { push @$jobs, $j; }
}
Expand Down Expand Up @@ -1974,8 +2007,10 @@ sub _get_annotation_md5 {
map { $data->{$mg}{$row[1]}{$_} = 1 } @{ $row[2] };
}
$sth->finish;
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
if ($self->_memd) {
foreach my $mg (keys %$data) {
$self->_memd->set($mg.$cache_key, $data->{$mg}, $self->_expire);
}
}
$self->_dbh->commit;
return $data;
Expand Down
2 changes: 1 addition & 1 deletion src/MGRAST/lib/resources/resource.pm
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ sub new {
eval {
require Cache::Memcached;
Cache::Memcached->import();
$memd = new Cache::Memcached {'servers' => [$Conf::web_memcache], 'debug' => 0, 'compress_threshold' => 10_000};
$memd = new Cache::Memcached {'servers' => [$Conf::web_memcache || ''], 'debug' => 0, 'compress_threshold' => 10_000};
};
my $agent = LWP::UserAgent->new;
my $json = JSON->new;
Expand Down
6 changes: 3 additions & 3 deletions src/MGRAST/lib/resources/validation.pm
Original file line number Diff line number Diff line change
Expand Up @@ -355,11 +355,11 @@ sub data {
# }
} else {
# getting node
my $template_node = $self->get_shock_node($Conf::mgrast_md_template_node_id, $Conf::shock_globus_token);
my $template_node = $self->get_shock_node($Conf::mgrast_md_template_node_id, $self->mgrast_token);
$template_attributes = $template_node->{attributes};

# getting file
my $template_str = $self->get_shock_file($Conf::mgrast_md_template_node_id, undef, $Conf::shock_globus_token);
my $template_str = $self->get_shock_file($Conf::mgrast_md_template_node_id, undef, $self->mgrast_token);
$template = $json->decode($template_str);
}
# check shock type to be template
Expand Down Expand Up @@ -610,7 +610,7 @@ sub reformat_template {
}
}

my $node = $self->set_shock_node("mgrast", undef, $template, $Conf::shock_globus_token);
my $node = $self->set_shock_node("mgrast", undef, $template, $self->mgrast_token);

return $self->return_data($node);
}
Expand Down