Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/Babel/bin/create_pg_tables.sql
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ functions integer
DROP TABLE IF EXISTS md5_protein;
CREATE TABLE md5_protein (
_id SERIAL PRIMARY KEY,
md5 integer REFERENCES md5s(_id),
md5 char(32) NOT NULL,
id text NOT NULL,
function integer REFERENCES functions(_id),
organism integer REFERENCES organisms_ncbi(_id),
Expand All @@ -66,7 +66,7 @@ source integer REFERENCES sources(_id)
DROP TABLE IF EXISTS md5_ontology;
CREATE TABLE md5_ontology (
_id SERIAL PRIMARY KEY,
md5 integer REFERENCES md5s(_id),
md5 char(32) NOT NULL,
id text NOT NULL,
function integer REFERENCES functions(_id),
source integer REFERENCES sources(_id)
Expand All @@ -75,17 +75,17 @@ source integer REFERENCES sources(_id)
DROP TABLE IF EXISTS md5_rna;
CREATE TABLE md5_rna (
_id SERIAL PRIMARY KEY,
md5 integer REFERENCES md5s(_id),
md5 char(32) NOT NULL,
id text NOT NULL,
function integer REFERENCES functions(_id),
organism integer REFERENCES organisms_ncbi(_id),
source integer REFERENCES sources(_id),
tax_rank integer,
tax_rank integer
);

DROP TABLE IF EXISTS md5_lca;
CREATE TABLE md5_lca (
md5 integer REFERENCES md5s(_id),
md5 char(32) NOT NULL,
tax_domain text,
tax_phylum text,
tax_class text,
Expand Down
150 changes: 74 additions & 76 deletions src/Babel/bin/dump_annotation_for_solr.pl
Original file line number Diff line number Diff line change
Expand Up @@ -38,66 +38,52 @@
my $dbh = DBI->connect("DBI:Pg:dbname=$dbname;host=$dbhost", $dbuser, '', {AutoCommit => 0});
unless ($dbh) { print STDERR "Error: " . $DBI::errstr . "\n"; exit 1; }

## skipping GO
print STDERR "Loading sources ...\n";
my %src_lookup = ();
$sth = $dbh->prepare("SELECT _id, name, type FROM sources");
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
$src_lookup{$row[0]}{source_id} = int($row[0]);
$src_lookup{$row[0]}{source} = $row[1];
$src_lookup{$row[0]}{type} = $row[2];
}
$sth->finish;
my $src_data = ['source_id','source','organization','description','type','url','email','link','title','version','download_date'];
my $src_sql = "SELECT _id AS source_id, name AS source, source AS organization, description, type, url, email, link, title, version, download_date FROM sources WHERE _id != 11";
my $src_lookup = $dbh->selectall_hashref($src_sql , "source_id");

print STDERR "Dumping sources ...\n";
open(SRC, ">$output.source") or die "Couldn't open $output.source for writing.\n";
print SRC to_json(lookup2data($src_lookup, $src_data, 'static_source'), {ascii => 1})."\n";
close(SRC);

print STDERR "Loading ontologies ...\n";
my %ont_lookup = ();
$sth = $dbh->prepare("SELECT id, level1, level2, level3, level4 FROM ontologies");
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
$ont_lookup{$row[0]}{level1} = $row[1];
$ont_lookup{$row[0]}{level2} = $row[2];
$ont_lookup{$row[0]}{level3} = $row[3];
$ont_lookup{$row[0]}{level4} = $row[4];
my $ont_sql = "SELECT id, level1, level2, level3, level4, source FROM ontologies";
my $ont_lookup = $dbh->selectall_hashref($ont_sql , "id");
foreach my $onts (grep {$_->{type} eq 'ontology'} values %$src_lookup) {
my $ont_data = ['id','level1','level2','level3','level4'];
if ($onts->{source} =~ /^[CN]OG$/) {
pop @$ont_data;
}
my %sub_ont_lookup = map {$_, $ont_lookup->{$_}} grep {$ont_lookup->{$_}{source} eq $onts->{source_id}} keys %$ont_lookup;
print STDERR "Dumping ontology ".$onts->{source}." ...\n";
open(TAX, ">$output.".$onts->{source}) or die "Couldn't open $output.".$onts->{source}." for writing.\n";
print TAX to_json(lookup2data(\%sub_ont_lookup, $ont_data, 'static_'.$onts->{source}), {ascii => 1})."\n";
close(TAX);
}
$sth->finish;

print STDERR "Loading organisms ...\n";
my %org_lookup = ();
$sth = $dbh->prepare("SELECT _id, name, tax_domain, tax_phylum, tax_class, tax_order, tax_family, tax_genus, tax_species, ncbi_tax_id FROM organisms_ncbi");
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
$org_lookup{$row[0]}{organism_id} = int($row[0]);
$org_lookup{$row[0]}{organism} = $row[1];
if ($row[2]) { $org_lookup{$row[0]}{domain} = $row[2]; }
if ($row[3]) { $org_lookup{$row[0]}{phylum} = $row[3]; }
if ($row[4]) { $org_lookup{$row[0]}{class} = $row[4]; }
if ($row[5]) { $org_lookup{$row[0]}{order} = $row[5]; }
if ($row[6]) { $org_lookup{$row[0]}{family} = $row[6]; }
if ($row[7]) { $org_lookup{$row[0]}{genus} = $row[7]; }
if ($row[8]) { $org_lookup{$row[0]}{species} = $row[8]; }
if ($row[9]) { $org_lookup{$row[0]}{ncbi_tax_id} = int($row[9]); }
}
$sth->finish;
my $org_data = ['organism_id','organism','domain','phylum','class','order','family','genus','species','ncbi_tax_id'];
my $org_sql = "SELECT _id AS organism_id, name AS organism, tax_domain AS domain, tax_phylum AS phylum, tax_class AS class, tax_order AS order, tax_family AS family, tax_genus AS genus, tax_species AS species, ncbi_tax_id FROM organisms_ncbi";
my $org_lookup = $dbh->selectall_hashref($org_sql , "organism_id");

print STDERR "Dumping taxonomy ...\n";
open(TAX, ">$output.taxonomy") or die "Couldn't open $output.taxonomy for writing.\n";
print TAX to_json(lookup2data($org_lookup, $org_data, 'static_taxonomy'), {ascii => 1})."\n";
close(TAX);

$dbh->disconnect;
exit;

print STDERR "Loading functions ...\n";
my %func_lookup = ();
$sth = $dbh->prepare("SELECT _id, name FROM functions");
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
$func_lookup{$row[0]}{function_id} = int($row[0]);
$func_lookup{$row[0]}{function} = $row[1];
}
$sth->finish;
my $func_sql = "SELECT _id AS function_id, name AS function FROM functions";
my $func_lookup = $dbh->selectall_hashref($func_sql , "function_id");

print STDERR "Loading md5s ...\n";
my %md5_lookup = ();
$sth = $dbh->prepare("SELECT _id, md5 FROM md5s");
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
$md5_lookup{$row[1]}{md5_id} = int($row[0]);
$md5_lookup{$row[1]}{md5} = $row[1];
}
$sth->finish;
my $md5_sql = "SELECT _id AS md5_id, md5 FROM md5s";
my $md5_lookup = $dbh->selectall_hashref($md5_sql , "md5_id");

open(DUMP, ">$output") or die "Couldn't open $output for writing.\n";
# rna
Expand All @@ -108,17 +94,17 @@
while (my @row = $sth->fetchrow_array()) {
my ($md5, $id, $func, $org, $src) = @row;
my $data = { 'id' => $count, 'accession' => $id };
if ($md5 && exists($md5_lookup{$md5})) {
map { $data->{$_} = $md5_lookup{$md5}{$_} } keys %{$md5_lookup{$md5}};
if ($md5 && exists($md5_lookup->{$md5})) {
map { $data->{$_} = $md5_lookup->{$md5}{$_} } keys %{$md5_lookup->{$md5}};
}
if ($func && exists($func_lookup{$func})) {
map { $data->{$_} = $func_lookup{$func}{$_} } keys %{$func_lookup{$func}};
if ($func && exists($func_lookup->{$func})) {
map { $data->{$_} = $func_lookup->{$func}{$_} } keys %{$func_lookup->{$func}};
}
if ($org && exists($org_lookup{$org})) {
map { $data->{$_} = $org_lookup{$org}{$_} } keys %{$org_lookup{$org}};
if ($org && exists($org_lookup->{$org})) {
map { $data->{$_} = $org_lookup->{$org}{$_} } keys %{$org_lookup->{$org}};
}
if ($src && exists($src_lookup{$src})) {
map { $data->{$_} = $src_lookup{$src}{$_} } keys %{$src_lookup{$src}};
if ($src && exists($src_lookup->{$src})) {
map { $data->{$_} = $src_lookup->{$src}{$_} } grep { exists $src_lookup->{$src}{$_} } ('source_id', 'source', 'type');
}
$count += 1;
print DUMP to_json($data, {ascii => 1})."\n";
Expand All @@ -132,46 +118,58 @@
while (my @row = $sth->fetchrow_array()) {
my ($md5, $id, $func, $org, $src) = @row;
my $data = { 'id' => $count, 'accession' => $id };
if ($md5 && exists($md5_lookup{$md5})) {
map { $data->{$_} = $md5_lookup{$md5}{$_} } keys %{$md5_lookup{$md5}};
if ($md5 && exists($md5_lookup->{$md5})) {
map { $data->{$_} = $md5_lookup->{$md5}{$_} } keys %{$md5_lookup->{$md5}};
}
if ($func && exists($func_lookup{$func})) {
map { $data->{$_} = $func_lookup{$func}{$_} } keys %{$func_lookup{$func}};
if ($func && exists($func_lookup->{$func})) {
map { $data->{$_} = $func_lookup->{$func}{$_} } keys %{$func_lookup->{$func}};
}
if ($org && exists($org_lookup{$org})) {
map { $data->{$_} = $org_lookup{$org}{$_} } keys %{$org_lookup{$org}};
if ($org && exists($org_lookup->{$org})) {
map { $data->{$_} = $org_lookup->{$org}{$_} } keys %{$org_lookup->{$org}};
}
if ($src && exists($src_lookup{$src})) {
map { $data->{$_} = $src_lookup{$src}{$_} } keys %{$src_lookup{$src}};
if ($src && exists($src_lookup->{$src})) {
map { $data->{$_} = $src_lookup->{$src}{$_} } grep { exists $src_lookup->{$src}{$_} } ('source_id', 'source', 'type');
}
$count += 1;
print DUMP to_json($data, {ascii => 1})."\n";
}
$sth->finish;

# ontology
# ontology - skip GO
print STDERR "Dumping ontology annotations ...\n";
$sth = $dbh->prepare("SELECT md5, id, function, source FROM md5_ontology WHERE source != 11");
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
my ($md5, $id, $func, $src) = @row;
my $data = { 'id' => $count, 'accession' => $id };
if ($md5 && exists($md5_lookup{$md5})) {
map { $data->{$_} = $md5_lookup{$md5}{$_} } keys %{$md5_lookup{$md5}};
if ($md5 && exists($md5_lookup->{$md5})) {
map { $data->{$_} = $md5_lookup->{$md5}{$_} } keys %{$md5_lookup->{$md5}};
}
if ($func && exists($func_lookup{$func})) {
map { $data->{$_} = $func_lookup{$func}{$_} } keys %{$func_lookup{$func}};
if ($func && exists($func_lookup->{$func})) {
map { $data->{$_} = $func_lookup->{$func}{$_} } keys %{$func_lookup->{$func}};
}
if (exists($ont_lookup{$id})) {
map { $data->{$_} = $ont_lookup{$id}{$_} } keys %{$ont_lookup{$id}};
if ($id && exists($ont_lookup->{$id})) {
map { $data->{$_} = $ont_lookup->{$id}{$_} } grep { exists $ont_lookup->{$id}{$_} } ('level1', 'level2', 'level3', 'level4');
}
if ($src && exists($src_lookup{$src})) {
map { $data->{$_} = $src_lookup{$src}{$_} } keys %{$src_lookup{$src}};
if ($src && exists($src_lookup->{$src})) {
map { $data->{$_} = $src_lookup->{$src}{$_} } grep { exists $src_lookup->{$src}{$_} } ('source_id', 'source', 'type');
}
$count += 1;
print DUMP to_json($data, {ascii => 1})."\n";
}
$sth->finish;

close(DUMP);
$dbh->disconnect;

sub lookup2data {
my ($lookup, $list, $id) = @_;
my %data = map {$_, []} @$list;
foreach my $set (values %$lookup) {
foreach my $field (@$list) {
my $value = exists($set->{$field}) ? $set->{$field} : undef;
push @{$data{$field}}, $value;
}
}
$data{id} = $id;
return \%data;
}
2 changes: 1 addition & 1 deletion src/MGRAST/html/js/MetagenomeSearch.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
var api_url = 'http://dev.metagenomics.anl.gov/api.cgi/metagenome?verbosity=mixs&';
var api_url = 'http://api.metagenomics.anl.gov/1/metagenome?verbosity=mixs&';
var datastore = {};
var result = 'result'; // div where results are to be displayed
var saved_params = {};
Expand Down
55 changes: 32 additions & 23 deletions src/MGRAST/lib/WebPage/DownloadMetagenome.pm
Original file line number Diff line number Diff line change
Expand Up @@ -536,32 +536,22 @@ sub api_download_builder {
<tr>
<td>
<select id='ann_type' onchange='
var asrc = undefined;
var sel_type = this.options[this.selectedIndex].value;
if (sel_type == "ontology") {
document.getElementById("ont_source").style.display="";
document.getElementById("org_source").style.display="none";
asrc = document.getElementById("ont_source");
} else {
document.getElementById("org_source").style.display="";
document.getElementById("ont_source").style.display="none";
asrc = document.getElementById("org_source");
}
var params = "type="+sel_type+"&source="+asrc.options[asrc.selectedIndex].value;
document.getElementById("api_url").innerHTML = "$sim?"+params;
document.getElementById("api_link").href = "metagenomics.cgi?page=DownloadMetagenome&action=api_download&mid=$mid&"+params;'>
}'>
<option>organism</option>
<option>function</option>
<option>ontology</option>
<option>feature</option>
</select></td>
<td>&nbsp;&nbsp;&nbsp;&nbsp;</td>
<td>
<select id='org_source' onchange='
var atype = document.getElementById("ann_type");
var params = "type="+atype.options[atype.selectedIndex].value+"&source="+this.options[this.selectedIndex].value;
document.getElementById("api_url").innerHTML = "$sim?"+params;
document.getElementById("api_link").href = "metagenomics.cgi?page=DownloadMetagenome&action=api_download&mid=$mid&"+params;'>
<select id='org_source'>
<option>RefSeq</option>
<option>GenBank</option>
<option>IMG</option>
Expand All @@ -575,22 +565,35 @@ sub api_download_builder {
<option>LSU</option>
<option>SSU</option>
</select>
<select id='ont_source' style='display:none;' onchange='
var atype = document.getElementById("ann_type");
var params = "type="+atype.options[atype.selectedIndex].value+"&source="+this.options[this.selectedIndex].value;
document.getElementById("api_url").innerHTML = "$sim?"+params;
document.getElementById("api_link").href = "metagenomics.cgi?page=DownloadMetagenome&action=api_download&mid=$mid&"+params;'>
<select id='ont_source' style='display:none;'>
<option>Subsystems</option>
<option>NOG</option>
<option>COG</option>
<option>KO</option>
</select></td>
<td>&nbsp;&nbsp;&nbsp;&nbsp;</td>
<td><a id='api_link' href='metagenomics.cgi?page=DownloadMetagenome&action=api_download&mid=$mid&$default'><b>Download</b></a></td>
<td><a id='api_link' href='#' onclick='
var asrc = undefined;
var atype = document.getElementById("ann_type");
var sel_type = atype.options[atype.selectedIndex].value;
if (sel_type == "ontology") {
asrc = document.getElementById("ont_source");
} else {
asrc = document.getElementById("org_source");
}
var params = "mid=$mid&type="+sel_type+"&source="+asrc.options[asrc.selectedIndex].value;
var url = "metagenomics.cgi?page=DownloadMetagenome&action=api_download&"+params;
if (document.getElementById("login_input_box") == null) {
var wsCookie = getCookie("WebSession");
if (wsCookie) {
url += "&auth=" + wsCookie;
}
}
this.href = url;'><b>Download</b>
</a></td>
</tr>
</table>);
#<p><b>URL:</b>&nbsp;&nbsp;<code id='api_url'>$sim?$default</code></p>
$html .= "</td></tr></table><br><br>";
</table>
</td></tr></table><br><br>);

return $html;
}
Expand All @@ -601,6 +604,7 @@ sub api_download {
use LWP::UserAgent;
my $agent = LWP::UserAgent->new;
my $mid = $self->application->cgi->param('mid');
my $auth = $self->application->cgi->param('auth') || undef;
my $type = $self->application->cgi->param('type');
my $src = $self->application->cgi->param('source');
my $url = $self->data('api')."/annotation/similarity/mgm".$mid."?type=".$type."&source=".$src;
Expand All @@ -609,7 +613,12 @@ sub api_download {
print "Content-Type:application/x-download\n";
print "Content-Disposition:attachment;filename=mgm".$mid."_".$type."_".$src.".tab\n\n";
eval {
my $get = $agent->get($url, ':read_size_hint' => 1024, ':content_cb' => sub{my ($chunk) = @_; print $chunk;});
my $get = undef;
if ($auth) {
$get = $agent->get($url, 'auth' => $auth, ':read_size_hint' => 1024, ':content_cb' => sub{my ($chunk) = @_; print $chunk;});
} else {
$get = $agent->get($url, ':read_size_hint' => 1024, ':content_cb' => sub{my ($chunk) = @_; print $chunk;});
}
$content = $get->content;
};
if ($@) {
Expand Down Expand Up @@ -697,5 +706,5 @@ sub public_project_download_table {
}

sub require_javascript {
return [ "$Conf::cgi_url/Html/pipeline.js" ];
return [ "$Conf::cgi_url/Html/pipeline.js", "$Conf::cgi_url/Html/MetagenomeSearch.js" ];
}
18 changes: 7 additions & 11 deletions src/MGRAST/lib/resources/m5nr.pm
Original file line number Diff line number Diff line change
Expand Up @@ -499,8 +499,8 @@ sub query {
# get results
my ($result, $total);
if ($type eq 'md5') {
my $md5s = $self->clean_md5($data);
($result, $total) = $self->solr_data($type, $md5s, $source, $offset, $limit, $order, 1);
my @md5s = map { $self->clean_md5($_) } @$data;
($result, $total) = $self->solr_data($type, \@md5s, $source, $offset, $limit, $order, 1);
} elsif ($type eq 'accession') {
($result, $total) = $self->solr_data($type, $data, undef, $offset, $limit, $order, 1);
} else {
Expand All @@ -513,15 +513,11 @@ sub query {
}

sub clean_md5 {
my ($self, $md5s) = @_;
my $clean = [];
foreach my $m (@$md5s) {
my $c = $m;
$c =~ s/[^a-zA-Z0-9]//g;
unless ($c && (length($c) == 32)) {
$self->return_data({"ERROR" => "invalid md5 was entered ($m)"}, 404);
}
push @$clean, $c;
my ($self, $md5) = @_;
my $clean = $md5;
$clean =~ s/[^a-zA-Z0-9]//g;
unless ($clean && (length($clean) == 32)) {
$self->return_data({"ERROR" => "invalid md5 was entered ($md5)"}, 404);
}
return $clean;
}
Expand Down