Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions src/MGRAST/Schema/MGRAST_JobDB.xml
Original file line number Diff line number Diff line change
Expand Up @@ -207,10 +207,11 @@
<scalar label="type" type="CHAR(32)" mandatory="1" />
<scalar label="tag" type="CHAR(128)" mandatory="1" />
<scalar label="value" type="TEXT" />
<unique_index>
<attribute label="type"/>
<attribute label="tag"/>
</unique_index>
<scalar label="value_id" type="TEXT" />
<scalar label="value_version" type="CHAR(32)" />
<index><attribute label="type"/></index>
<index><attribute label="tag"/></index>
<index><attribute label="value_version"/></index>
</object>

<object label="Measurement">
Expand Down
122 changes: 68 additions & 54 deletions src/MGRAST/lib/Metadata.pm
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,7 @@ sub new {

$app = $app || '';
$debug = $debug || '';
my $self = { app => $app,
debug => $debug,
ontology_url => 'http://bioportal.bioontology.org/visualize/',
ontology_api => 'http://rest.bioontology.org/bioportal/search',
ontology_key => '56a9721b-0d62-4185-933d-81447db2457a'
};
my $self = { app => $app, debug => $debug };
eval {
$self->{_handle} = DBMaster->new( -database => $Conf::mgrast_jobcache_db || 'MGRASTMetadata',
-host => $Conf::mgrast_jobcache_host,
Expand All @@ -45,7 +40,7 @@ sub new {

Returns the metadata temple in the format:
category => 'category_type' => <category_type>
category => tag => { mgrast_tag, qiime_tag, definition, type, fw_type, required, mixs }
category => tag => { aliases, definition, type, required, mixs, unit }

=cut

Expand All @@ -55,35 +50,67 @@ sub template {
unless ($self->{data} && ref($self->{data})) {
my $data = {};
my $dbh = $self->{_handle}->db_handle;
my $tmp = $dbh->selectall_arrayref("SELECT category_type,category,tag,mgrast_tag,qiime_tag,definition,type,fw_type,required,mixs,unit FROM MetaDataTemplate");
my $tmp = $dbh->selectall_arrayref("SELECT category_type,category,tag,mgrast_tag,qiime_tag,definition,required,type,mixs,unit FROM MetaDataTemplate");
unless ($tmp && (@$tmp)) { return $data; }
map { $data->{$_->[1]}{category_type} = $_->[0] } @$tmp;
map { $data->{$_->[1]}{$_->[2]} = { mgrast_tag => $_->[3],
qiime_tag => $_->[4],
definition => $_->[5],
type => $_->[6],
fw_type => $_->[7],
required => $_->[8],
mixs => $_->[9],
unit => $_->[10]
} } @$tmp;
map { $data->{$_->[1]}{$_->[2]} =
{ aliases => [ $_->[3], $_->[4] ],
definition => $_->[5],
required => $_->[6],
type => $_->[7],
mixs => $_->[8],
unit => $_->[9]
} } @$tmp;
$self->{data} = $data;
}
return $self->{data};
}

sub get_cv_list {
my ($self, $tag) = @_;
sub get_cv_select {
my ($self, $tag, $version) = @_;
if (! $version) {
$version = $self->cv_latest_version($tag);
}
my $dbh = $self->{_handle}->db_handle;
my $tmp = $dbh->selectcol_arrayref("SELECT value FROM MetaDataCV WHERE tag='$tag' AND type='select' AND value_version='$version'");
return ($tmp && @$tmp) ? $tmp : [];
}

sub get_cv_ontology {
my ($self, $tag, $version) = @_;
if (! $version) {
$version = $self->cv_latest_version($tag);
}
my $dbh = $self->{_handle}->db_handle;
my $tmp = $dbh->selectcol_arrayref("SELECT value FROM MetaDataCV WHERE tag='$tag' AND type='select'");
my $tmp = $dbh->selectall_arrayref("SELECT value, value_id FROM MetaDataCV WHERE tag='$tag' AND type='ontology' AND value_version='$version'");
return ($tmp && @$tmp) ? $tmp : [];
}

sub get_ont_id {
sub cv_ontology_info {
my ($self, $tag, $version) = @_;
if (! $version) {
$version = $self->cv_latest_version($tag);
}
my $dbh = $self->{_handle}->db_handle;
my $tmp = $dbh->selectrow_arrayref("SELECT value, value_id FROM MetaDataCV WHERE tag='$tag' AND type='ont_info' AND value_version='$version'");
return ($tmp && @$tmp) ? $tmp : ['', ''];
}

sub cv_latest_version {
my ($self, $tag) = @_;
my $dbh = $self->{_handle}->db_handle;
my $tmp = $dbh->selectcol_arrayref("SELECT value FROM MetaDataCV WHERE tag='$tag' AND type='ontology'");
return ($tmp && @$tmp) ? $tmp->[0] : '';
if ($tag) {
my $one = $dbh->selectcol_arrayref("SELECT value FROM MetaDataCV WHERE tag='$tag' AND type='latest_version'");
return ($one && @$one) ? $one->[0] : '';
} else {
my $all = $dbh->selectall_arrayref("SELECT tag, value FROM MetaDataCV WHERE type='latest_version'");
if ($all && @$all) {
my %data = map {$_->[0], $_->[1]} @$all;
return \%data;
} else {
return {};
}
}
}

=pod
Expand Down Expand Up @@ -158,7 +185,7 @@ return tuple ref: [ boolean, error_message ]
=cut

sub validate_value {
my ($self, $cat, $tag, $val) = @_;
my ($self, $cat, $tag, $val, $version) = @_;

my $dbh = $self->{_handle}->db_handle;
my $tmp = $dbh->selectcol_arrayref("SELECT distinct type FROM MetaDataTemplate WHERE category_type='$cat' AND tag='$tag'");
Expand All @@ -181,36 +208,27 @@ sub validate_value {
} elsif ($type eq 'timezone') {
return ($val =~ /^UTC/) ? [1, ''] : [0, 'not ISO8601 compliant timezone'];
} elsif ($type eq 'select') {
my %cvs = map {$_, 1} @{ $self->get_cv_list($tag) };
# hack for different country tags
if ($tag =~ /country$/) {
$tag = 'country';
}
my %cvs = map {$_, 1} @{ $self->get_cv_select($tag, $version) };
return exists($cvs{lc($val)}) ? [1, ''] : [0, 'not one of: '.join(', ', sort keys %cvs)];
} elsif ($type eq 'ontology') {
my $check = $self->validate_ontology($tag, $val);
return ($check eq 'valid') ? [1, ''] : [0, $check];
my $cvi = $self->cv_ontology_info($tag, $version);
my $cvo = $self->get_cv_ontology($tag, $version);
my %oid = map {$_->[1], 1} @$cvo;
my %oname = map {$_->[0], 1} @$cvo;
if (exists($oname{$val}) || exists($oid{$val})) {
return [1, '']
} else {
return [0, 'not part of: '.$cvi->[0]];
}
} else {
return [0, 'unknown type'];
}
}

sub validate_ontology {
my ($self, $tag, $val) = @_;

my $oid = $self->get_ont_id($tag);
unless ($oid) {
return 'not ontology label: '.$tag;
}
my $url = $self->{ontology_api}.'?isexactmatch=1&apikey='.$self->{ontology_key}.'&ontologyids='.$oid.'&query='.uri_escape($val);
my $res = get($url);
unless ($res && ($res =~ /<success>/)) {
return 'bioportal inaccessable: can not connect to: '.$url;
}
if ($res =~ /<numHitsTotal>(\d+)<\/numHitsTotal>/) {
return ($1 > 0) ? 'valid' : 'not part of: '.$self->{ontology_url}.$oid;
}
else {
return "ontology ID $oid has malformed return structure from ".$url;
}
}

=pod

=item * B<get_jobs_metadata_fast> (Array<ids>, Boolean<is_mgid>)
Expand Down Expand Up @@ -651,7 +669,7 @@ sub export_metadata_for_project {
}

## input: { tag => value }
## output: { tag => {qiime_tag=><str>, mgrast_tag=><str>, definition=><str>, required=><bool>, mixs=><bool>, type=><str>, value=><str>} }
## output: { tag => {aliases=>[<str>], definition=><str>, required=><bool>, mixs=><bool>, type=><str>, unit=><str>, value=><str>} }
## all required tags will be added
sub add_template_to_data {
my ($self, $cat, $data, $all) = @_;
Expand Down Expand Up @@ -686,13 +704,9 @@ sub add_template_to_data {
$t_data->{$tag}{unit} = $template->{$cat}{$ttag}{unit};
$t_data->{$tag}{type} = $template->{$cat}{$ttag}{type};
$t_data->{$tag}{mixs} = $template->{$cat}{$ttag}{mixs};
$t_data->{$tag}{required} = $template->{$cat}{$ttag}{required};
$t_data->{$tag}{aliases} = $template->{$cat}{$ttag}{aliases};
$t_data->{$tag}{required} = $template->{$cat}{$ttag}{required};
$t_data->{$tag}{definition} = $template->{$cat}{$ttag}{definition};
foreach my $alias (($template->{$cat}{$ttag}{mgrast_tag}, $template->{$cat}{$ttag}{qiime_tag})) {
if ($alias && ($alias ne $tag)) {
push @{ $t_data->{$tag}{aliases} }, $alias;
}
}
}
return $t_data;
}
Expand Down
107 changes: 87 additions & 20 deletions src/MGRAST/lib/resources/annotation.pm
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ no warnings('once');
use List::MoreUtils qw(any uniq);
use Data::Dumper;
use HTML::Strip;
use URI::Escape;

use Conf;
use parent qw(resources::resource);
Expand All @@ -23,13 +24,16 @@ sub new {
$self->{types} = [[ "organism", "return organism data" ],
[ "function", "return function data" ],
[ "ontology", "return ontology data" ],
[ "feature", "return feature data" ]];
$self->{cutoffs} = { evalue => '5', identity => '60', length => '15' };
[ "feature", "return feature data" ],
[ "md5", "return md5sum data" ]];
$self->{cutoffs} = { evalue => '5', identity => '60', length => '15' };
$self->{m5nr_ver} = "1";
$self->{m5nr_max} = 10000;
$self->{attributes} = { sequence => {
"col_01" => ['string', 'sequence id'],
"col_02" => ['string', 'm5nr id (md5sum)'],
"col_03" => ['string', 'semicolon seperated list of annotations'],
"col_04" => ['string', 'dna sequence'] },
"col_03" => ['string', 'dna sequence'],
"col_04" => ['string', 'semicolon seperated list of annotations'] },
similarity => {
"col_01" => ['string', 'query sequence id'],
"col_02" => ['string', 'hit m5nr id (md5sum)'],
Expand Down Expand Up @@ -160,13 +164,50 @@ sub prepare_data {

my $cgi = $self->cgi;
my $type = $cgi->param('type') ? $cgi->param('type') : 'organism';
my $filter = $cgi->param('filter') || undef;
my $flevel = $cgi->param('filter_level') || undef;
my $source = $cgi->param('source') ? $cgi->param('source') : (($type eq 'ontology') ? 'Subsystems' : 'RefSeq');
my $eval = defined($cgi->param('evalue')) ? $cgi->param('evalue') : $self->{cutoffs}{evalue};
my $ident = defined($cgi->param('identity')) ? $cgi->param('identity') : $self->{cutoffs}{identity};
my $alen = defined($cgi->param('length')) ? $cgi->param('length') : $self->{cutoffs}{length};

my $filter = $cgi->param('filter') || undef;
my $flevel = $cgi->param('filter_level') || undef;
my $md5s = [];

# post of md5s
if ($self->method eq 'POST') {
my $post_data = $self->cgi->param('POSTDATA') ? $self->cgi->param('POSTDATA') : join(" ", $self->cgi->param('keywords'));
# all options sent as post data
if ($post_data) {
eval {
my $json_data = $self->json->decode($post_data);
if (exists $json_data->{type}) { $type = $json_data->{type}; }
if (exists $json_data->{source}) { $source = $json_data->{source}; }
if (exists $json_data->{evalue}) { $eval = $json_data->{evalue}; }
if (exists $json_data->{identity}) { $ident = $json_data->{identity}; }
if (exists $json_data->{length}) { $alen = $json_data->{length}; }
if ($type eq 'md5') {
$filter = undef;
$flevel = undef;
$md5s = $json_data->{md5s};
}
};
# data sent in post form
} elsif ($self->cgi->param('md5s')) {
eval {
@$md5s = split(/;/, $self->cgi->param('md5s'));
};
} else {
$self->return_data( {"ERROR" => "POST request missing md5s"}, 400 );
}
if ($@ || (@$md5s == 0)) {
$self->return_data( {"ERROR" => "unable to obtain POSTed data: ".$@}, 500 );
}
} elsif ($filter && ($type eq 'md5')) {
$filter = undef;
$flevel = undef;
$md5s = [$filter];
}

# set DB handle
my $master = $self->connect_to_datasource();
use MGRAST::Analysis;
my $mgdb = MGRAST::Analysis->new( $master->db_handle );
Expand All @@ -176,6 +217,7 @@ sub prepare_data {
my $mgid = $data->{metagenome_id};
$mgdb->set_jobs([$mgid]);

# validate options
unless (exists $mgdb->_src_id->{$source}) {
$self->return_data({"ERROR" => "Invalid source was entered ($source). Please use one of: ".join(", ", keys %{$mgdb->_src_id})}, 404);
}
Expand All @@ -188,7 +230,9 @@ sub prepare_data {
unless (any {$_->[0] eq $type} @{$self->{types}}) {
$self->return_data({"ERROR" => "Invalid type was entered ($type). Please use one of: ".join(", ", map {$_->[0]} @{$self->{types}})}, 404);
}
if ( ($flevel =~ /^strain|species|function$/) || ($type !~ /^organism|ontology$/) ) {

# only have filter_level for organism or ontology
if ( $flevel && (($flevel =~ /^strain|species|function$/) || ($type !~ /^organism|ontology$/)) ) {
$flevel = undef;
}
if ($filter && $flevel) {
Expand All @@ -197,14 +241,31 @@ sub prepare_data {
$self->return_data({"ERROR" => "Invalid filter_level was entered ($flevel). For organism use one of: ".join(", ", map {$_->[0]} @{$self->hierarchy->{organism}}).". For ontology use one of: ".join(", ", map {$_->[0]} @{$self->hierarchy->{ontology}})}, 404);
}
}


# build queries
$eval = (defined($eval) && ($eval =~ /^\d+$/)) ? "exp_avg <= " . ($eval * -1) : "";
$ident = (defined($ident) && ($ident =~ /^\d+$/)) ? "ident_avg >= $ident" : "";
$alen = (defined($alen) && ($alen =~ /^\d+$/)) ? "len_avg >= $alen" : "";

my $query = "";
if (@$md5s) {
$query = "SELECT j.md5, j.seek, j.length FROM ".$mgdb->_jtbl->{md5}." j, ".$mgdb->_atbl->{md5}." m";
$query .= $mgdb->_get_where_str([ 'j.'.$mgdb->_qver, "j.job = ".$data->{job_id},
'j.'.$eval, 'j.'.$ident, 'j.'.$alen,
'j.seek IS NOT NULL', 'j.length IS NOT NULL',
'j.md5 = m._id', 'm.md5 IN ('.join(",", map {$mgdb->_dbh->quote($_)} @$md5s).')'
]);
$query .= " ORDER BY j.seek";
} else {
$query = "SELECT md5, seek, length FROM ".$mgdb->_jtbl->{md5};
$query .= $mgdb->_get_where_str([ $mgdb->_qver, "job = ".$data->{job_id},
$eval, $ident, $alen,
"seek IS NOT NULL", "length IS NOT NULL"
]);
$query .= " ORDER BY seek";
}

my $srcid = $mgdb->_src_id->{$source};
my $where = $mgdb->_get_where_str([$mgdb->_qver, "job = ".$data->{job_id}, $eval, $ident, $alen, "seek IS NOT NULL", "length IS NOT NULL"]);
my $query = "SELECT md5, seek, length FROM ".$mgdb->_jtbl->{md5}.$where." ORDER BY seek";
my @head = map { $self->{attributes}{$format}{$_}[1] } sort keys %{$self->{attributes}{$format}};
my $count = 0;

Expand All @@ -218,8 +279,9 @@ sub prepare_data {

# loop through indices and print data
while (my @row = $sth->fetchrow_array()) {
my ($md5, $seek, $len) = @row;
my ($md5, $seek, $len, $mmd5) = @row;
my $sql = "";
my $ann = [];
if ($type eq 'organism') {
$sql = "SELECT DISTINCT o.name FROM md5_annotation a, organisms_ncbi o WHERE a.md5=$md5 AND a.source=$srcid AND a.organism=o._id";
if ($filter && $flevel) {
Expand All @@ -232,17 +294,19 @@ sub prepare_data {
if ($filter && $flevel) {
$sql .= " AND o.".$flevel."=".$mgdb->_dbh->quote($filter);
}
} else {
} elsif ($type eq 'feature') {
$sql = "SELECT DISTINCT id FROM md5_annotation WHERE md5=$md5 AND source=$srcid";
}
my $ann = $mgdb->_dbh->selectcol_arrayref($sql);

# remove non-matching annotations if using filter without hierarchal level
if ($filter && (! $flevel)) {
my @matches = grep {/$filter/} @$ann;
@$ann = @matches;
if ($type ne 'md5') {
$ann = $mgdb->_dbh->selectcol_arrayref($sql);
# remove non-matching annotations if using filter without hierarchal level
if ($filter && (! $flevel)) {
my @matches = grep {/$filter/} @$ann;
@$ann = @matches;
}
if (@$ann == 0) { next; }
}
if (@$ann == 0) { next; }

# pull data from indexed file
my $rec = '';
Expand All @@ -259,11 +323,14 @@ sub prepare_data {
}
$hs->eof;
if (($format eq 'sequence') && (@tabs == 13)) {
@out = ('mgm'.$mgid."|".$rid, $tabs[1], join(";", @$ann), $tabs[12]);
@out = ('mgm'.$mgid."|".$rid, $tabs[1], $tabs[12], join(";", @$ann));
} elsif ($format eq 'similarity') {
@out = ('mgm'.$mgid."|".$rid, @tabs[1..11], join(";", @$ann));
$count += 1;
}
if ($type eq 'md5') {
pop @out;
}
print join("\t", map {$_ || ''} @out)."\n";
$count += 1;
}
Expand Down
Loading