Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/MGRAST/Schema/m5nr_table.cql.tt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ CREATE KEYSPACE IF NOT EXISTS m5nr_v[% version %]

use m5nr_v[% version %];

CREATE TABLE IF NOT EXISTS index_annotation (
id int,
source text,
md5 text,
is_protein boolean,
single int,
accession list<int>,
function list<int>,
organism list<int>,
PRIMARY KEY (id, source)
);

CREATE TABLE IF NOT EXISTS id_annotation (
id int,
source text,
Expand Down
48 changes: 48 additions & 0 deletions src/MGRAST/lib/Abundance.pm
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,32 @@ use DBI;
use Data::Dumper;
use List::Util qw(max min sum);

our $version = $Conf::m5nr_annotation_version || 1;
our $ontology_md5s = {}; # src => ont => [md5s]
our $function_md5s = {}; # func => [md5s]
our $organism_md5s = {}; # org => [md5s]
our $md5_abundance = {}; # md5 => abund
our $dbh = undef;

sub get_where_str {
my ($items) = @_;

my @text;
unless ($items && (@$items > 0)) { return ""; }
foreach my $i (@$items) {
if ($i && ($i =~ /\S/)) {
push @text, $i;
}
}
if (@text == 1) {
return " WHERE " . $text[0];
} elsif (@text > 1) {
return " WHERE " . join(" AND ", @text);
} else {
return "";
}
}

sub get_analysis_dbh {
eval {
# need write DB as read DB may not yet be synched
Expand All @@ -31,6 +51,34 @@ sub get_analysis_dbh {
}
}

sub execute_query {
my ($query) = @_;
unless ($dbh) {
# set dbh if undef
eval {
my $host = $Conf::mgrast_write_dbhost;
my $database = $Conf::mgrast_db;
my $user = $Conf::mgrast_dbuser;
my $password = $Conf::mgrast_dbpass;
$dbh = DBI->connect("DBI:Pg:dbname=$database;host=$host", $user, $password,
{ RaiseError => 1, AutoCommit => 0, PrintError => 0 }) ||
die $DBI::errstr;
};
if ($@) {
warn "Unable to connect to metagenomics database: $@\n";
}
}
my $sth = $dbh->prepare($query);
$sth->execute() or die "Couldn't execute statement: ".$sth->errstr;
return $sth;
}

sub end_query {
my ($sth) = @_;
$sth->finish;
$dbh->commit;
}

sub get_ontology_abundances {
my ($job, $ver) = @_;

Expand Down
72 changes: 37 additions & 35 deletions src/MGRAST/lib/resources/annotation.pm
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use HTML::Strip;
use URI::Escape;

use Conf;
use MGRAST::Abundance;
use parent qw(resources::resource);

# Override parent constructor
Expand Down Expand Up @@ -170,7 +171,8 @@ sub prepare_data {
my $alen = defined($cgi->param('length')) ? $cgi->param('length') : $self->{cutoffs}{length};
my $filter = $cgi->param('filter') || undef;
my $flevel = $cgi->param('filter_level') || undef;
my $md5s = [];
my $md5s = [];
my $mgid = 'mgm'.$data->{metagenome_id};

# post of md5s
if ($self->method eq 'POST') {
Expand Down Expand Up @@ -206,26 +208,16 @@ sub prepare_data {
$flevel = undef;
$md5s = [$filter];
}

# set DB handle
my $master = $self->connect_to_datasource();
use MGRAST::Analysis;
my $mgdb = MGRAST::Analysis->new( $master->db_handle );
unless (ref($mgdb)) {
$self->return_data({"ERROR" => "resource database offline"}, 503);
}
my $mgid = $data->{metagenome_id};
$mgdb->set_jobs([$mgid]);

# validate options
unless (exists $mgdb->_src_id->{$source}) {
$self->return_data({"ERROR" => "Invalid source was entered ($source). Please use one of: ".join(", ", keys %{$mgdb->_src_id})}, 404);
unless ($self->valid_source($source)) {
$self->return_data({"ERROR" => "Invalid source was entered ($source). Please use one of: ".join(", ", @{$self->source_by_type()})}, 404);
}
if (($type eq 'ontology') && (! any {$_->[0] eq $source} @{$self->source->{ontology}})) {
$self->return_data({"ERROR" => "Invalid ontology source was entered ($source). Please use one of: ".join(", ", map {$_->[0]} @{$self->source->{ontology}})}, 404);
$self->return_data({"ERROR" => "Invalid ontology source was entered ($source). Please use one of: ".join(", ", @{$self->source_by_type('ontology')})}, 404);
}
if (($type eq 'organism') && (any {$_->[0] eq $source} @{$self->source->{ontology}})) {
$self->return_data({"ERROR" => "Invalid organism source was entered ($source). Please use one of: ".join(", ", map {$_->[0]} (@{$self->source->{protein}}, @{$self->source->{rna}}))}, 404);
$self->return_data({"ERROR" => "Invalid organism source was entered ($source). Please use one of: ".join(", ", (@{$self->source_by_type('protein')}, @{$self->source_by_type('rna')}))}, 404);
}
unless (any {$_->[0] eq $type} @{$self->{types}}) {
$self->return_data({"ERROR" => "Invalid type was entered ($type). Please use one of: ".join(", ", map {$_->[0]} @{$self->{types}})}, 404);
Expand All @@ -249,24 +241,35 @@ sub prepare_data {

my $query = "";
if (@$md5s) {
$query = "SELECT j.md5, j.seek, j.length FROM ".$mgdb->_jtbl->{md5}." j, ".$mgdb->_atbl->{md5}." m";
$query .= $mgdb->_get_where_str([ 'j.'.$mgdb->_qver, "j.job = ".$data->{job_id},
'j.'.$eval, 'j.'.$ident, 'j.'.$alen,
'j.seek IS NOT NULL', 'j.length IS NOT NULL',
'j.md5 = m._id', 'm.md5 IN ('.join(",", map {$mgdb->_dbh->quote($_)} @$md5s).')'
]);
$query = "SELECT j.md5, j.seek, j.length FROM job_md5s j, md5s m";
$query .= MGRAST::Abundance::get_where_str([
'j.version = '.$MGRAST::Abundance::version,
'j.job = '.$data->{job_id},
$eval ? 'j.'.$eval : "",
$ident ? 'j.'.$ident : "",
$alen ? 'j.'.$alen : "",
'j.seek IS NOT NULL',
'j.length IS NOT NULL',
'j.md5 = m._id',
'm.md5 IN ('.join(",", map {$MGRAST::Abundance::dbh->quote($_)} @$md5s).')'
]);
$query .= " ORDER BY j.seek";
} else {
$query = "SELECT md5, seek, length FROM ".$mgdb->_jtbl->{md5};
$query .= $mgdb->_get_where_str([ $mgdb->_qver, "job = ".$data->{job_id},
$eval, $ident, $alen,
"seek IS NOT NULL", "length IS NOT NULL"
]);
$query = "SELECT md5, seek, length FROM job_md5s";
$query .= MGRAST::Abundance::get_where_str([
'version = '.$MGRAST::Abundance::version,
'job = '.$data->{job_id},
$eval,
$ident,
$alen,
"seek IS NOT NULL",
"length IS NOT NULL"
]);
$query .= " ORDER BY seek";
}

# get shock node for file
my $params = {type => 'metagenome', data_type => 'similarity', stage_name => 'filter.sims', id => 'mgm'.$mgid};
my $params = {type => 'metagenome', data_type => 'similarity', stage_name => 'filter.sims', id => $mgid};
my $sim_node = $self->get_shock_query($params, $self->mgrast_token);
unless ((@$sim_node > 0) && exists($sim_node->[0]{id})) {
$self->return_data({"ERROR" => "Unable to retrieve $format file"}, 500);
Expand All @@ -285,7 +288,7 @@ sub prepare_data {
print join("\t", @head)."\n";

# get cassandra handle / prepare statement
my $chdl = $self->cassandra_m5nr_handle("m5nr_v".$mgdb->_version, $Conf::cassandra_m5nr);
my $chdl = $self->cassandra_m5nr_handle("m5nr_v".$MGRAST::Abundance::version, $Conf::cassandra_m5nr);
# get filter list
# filter_list is all taxa names that match filter for given filter_level (organism, ontology only)
my %filter_list = ();
Expand All @@ -298,9 +301,9 @@ sub prepare_data {
}

# start query
my $sth = $mgdb->_dbh->prepare($query);
$sth->execute() or die "Couldn't execute statement: ".$sth->errstr;

MGRAST::Abundance::get_analysis_dbh();
my $sth = MGRAST::Abundance::execute_query($query);
# loop through indexes and print data
my $count = 0;
my $md5_set = {};
Expand All @@ -323,8 +326,7 @@ sub prepare_data {
}

# cleanup
$sth->finish;
$mgdb->_dbh->commit;
MGRAST::Abundance::end_query($sth);
$chdl->close();
print "Download complete. $count rows retrieved\n";
exit 0;
Expand Down Expand Up @@ -389,9 +391,9 @@ sub print_batch {
my $rid = $tabs[0];
unless ($rid) { next; }
if (($format eq 'sequence') && (@tabs == 13)) {
@out = ('mgm'.$mgid."|".$rid, $tabs[1], $tabs[12], join(";", @$ann));
@out = ($mgid."|".$rid, $tabs[1], $tabs[12], join(";", @$ann));
} elsif ($format eq 'similarity') {
@out = ('mgm'.$mgid."|".$rid, @tabs[1..11], join(";", @$ann));
@out = ($mgid."|".$rid, @tabs[1..11], join(";", @$ann));
}
if ($type eq 'md5') {
pop @out;
Expand Down
35 changes: 24 additions & 11 deletions src/MGRAST/lib/resources/m5nr.pm
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,6 @@ sub new {
$self->{default} = '10';
$self->{request} = { ontology => 1, taxonomy => 1, sources => 1, accession => 1,
md5 => 1, function => 1, organism => 1, sequence => 1 };
$self->{version} = {
'1' => '20100309',
'9' => '20130801',
'10' => '20131215'
};
$self->{attributes} = { taxonomy => { data => [ 'list', ['object', [{'organism' => [ 'string', 'organism name' ],
'species' => [ 'string', 'organism species' ],
'genus' => [ 'string', 'organism genus' ],
Expand Down Expand Up @@ -112,7 +107,8 @@ sub info {
'filter' => ['string', 'text of ontology group (filter_level) to filter by'],
'min_level' => ['cv', $self->hierarchy->{ontology}],
'exact' => ['boolean', "if true return only those ontologies that exactly match filter, default is false"],
'version' => ['integer', 'M5NR version, default '.$self->{default}]
'version' => ['integer', 'M5NR version, default '.$self->{default}],
'compressed' => ['boolean', 'if true, return full compressed ontology, other options ignored'],
},
'required' => {},
'body' => {} }
Expand All @@ -130,7 +126,8 @@ sub info {
'filter' => ['string', 'text of taxonomy group (filter_level) to filter by'],
'min_level' => ['cv', [ @{$self->hierarchy->{organism}}[1..7] ]],
'exact' => ['boolean', "if true return only those taxonomies that exactly match filter, default is false"],
'version' => ['integer', 'M5NR version, default '.$self->{default}]
'version' => ['integer', 'M5NR version, default '.$self->{default}],
'compressed' => ['boolean', 'if true, return full compressed taxonomy, other options ignored'],
},
'required' => {},
'body' => {} }
Expand Down Expand Up @@ -419,16 +416,32 @@ sub static {
my $url = $self->{url}.'/m5nr/'.$type;
my $solr = 'object%3A';
my $limit = 1000000;
my $exact = $self->cgi->param('exact') ? 1 : 0;
my $exact = $self->cgi->param('exact') ? 1 : 0;
my $filter = $self->cgi->param('filter') || '';
my $min_lvl = $self->cgi->param('min_level') || '';
my $version = $self->cgi->param('version') || $self->{default};
my $compressed = $self->cgi->param('compressed') ? 1 : 0;
my $fields = [];
my $grouped = 0;

# validate version
$self->check_version($version);

# stream full compressed version from shock
if ($compressed && (($type eq 'ontology') || ($type eq 'taxonomy'))) {
my $query = {
type => 'reference',
data_type => 'm5nr hierarchy',
name => $type,
version => $version
};
my $nodes = $self->get_shock_query($query, $self->mgrast_token);
if (scalar(@$nodes) != 1) {
$self->return_data({"ERROR" => "missing compressed $type hierarchy for version $version"}, 404)
}
$self->return_shock_file($nodes->[0]{id}, $nodes->[0]{file}{size}, $nodes->[0]{file}{name}, $self->mgrast_token);
}

# return cached if exists
$self->return_cached();

Expand Down Expand Up @@ -654,7 +667,7 @@ sub md5s2sequences {
if ($Conf::m5nr_fasta && (-f $Conf::m5nr_fasta)) {
$m5nr = $Conf::m5nr_fasta;
} elsif ($Conf::m5nr_dir && (-d $Conf::m5nr_dir)) {
$m5nr = $Conf::m5nr_dir."/".$self->{version}{$version}."/md5nr";
$m5nr = $Conf::m5nr_dir."/".$self->{m5nr_version}{$version}."/md5nr";
} else {
$self->return_data({"ERROR" => "missing M5NR sequence data"}, 500);
}
Expand Down Expand Up @@ -692,8 +705,8 @@ sub md5s2sequences {

sub check_version {
my ($self, $version) = @_;
unless (exists $self->{version}{$version}) {
$self->return_data({"ERROR" => "invalid version was entered ($version). Please use one of: ".join(", ", keys %{$self->{version}})}, 404);
unless (exists $self->{m5nr_version}{$version}) {
$self->return_data({"ERROR" => "invalid version was entered ($version). Please use one of: ".join(", ", keys %{$self->{m5nr_version}})}, 404);
}
}

Expand Down
12 changes: 0 additions & 12 deletions src/MGRAST/lib/resources/metagenome.pm
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ use List::Util qw(first max min sum);
use POSIX qw(strftime floor);

use MGRAST::Metadata;
use MGRAST::Analysis;
use Conf;
use parent qw(resources::resource);

Expand All @@ -23,7 +22,6 @@ sub new {
# Add name / attributes
my %rights = $self->user ? map {$_, 1} grep {$_ ne '*'} @{$self->user->has_right_to(undef, 'view', 'metagenome')} : ();
$self->{name} = "metagenome";
$self->{mgdb} = undef;
$self->{rights} = \%rights;
$self->{cv} = {
verbosity => {'minimal' => 1, 'mixs' => 1, 'metadata' => 1, 'stats' => 1, 'full' => 1, 'seqstats' => 1},
Expand Down Expand Up @@ -447,16 +445,6 @@ sub prepare_data {
$mddb = MGRAST::Metadata->new();
$jobdata = $mddb->get_jobs_metadata_fast($mgids, 1);
}
if (($verb eq 'stats') || ($verb eq 'full')) {
# initialize analysis obj with mgids
my $master = $self->connect_to_datasource();
my $mgdb = MGRAST::Analysis->new( $master->db_handle );
unless (ref($mgdb)) {
$self->return_data({"ERROR" => "could not connect to analysis database"}, 500);
}
$mgdb->set_jobs($mgids);
$self->{mgdb} = $mgdb;
}

my $objects = [];
foreach my $job (@$data) {
Expand Down
Loading