Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
241 commits
Select commit Hold shift + click to select a range
329c6e9
Merge pull request #368 from MG-RAST/develop
teharrison Jul 3, 2013
c697bb9
Merge branch 'api' of https://github.com/MG-RAST/MG-RAST into develop
jaredbischof Aug 15, 2013
27ba7e1
Don't need to build api-doc target every time we run make.
jaredbischof Aug 16, 2013
05d3587
Merge pull request #434 from teharrison/develop
teharrison Aug 26, 2013
f431283
exchanged newsfeed for a working one
Aug 27, 2013
ee63329
added link to the technical manual
Aug 27, 2013
00f4d3b
updated for solr schema
Aug 27, 2013
6760f09
updated to only use solr (not SQL or Babel)
Aug 27, 2013
4b03f4e
api: more m5nr updates
Aug 27, 2013
83f2504
fixed bad format
Aug 28, 2013
94f816c
api: now able to query grouped solr response
Aug 28, 2013
7ab6c47
update for using only solr
Aug 28, 2013
7748c86
m5tool: fixed help option
Aug 28, 2013
a0d1d1a
Merge pull request #437 from teharrison/develop
teharrison Aug 28, 2013
3a7a373
eval'ed Auth usage
Aug 28, 2013
d3635fd
Merge pull request #438 from teharrison/develop
teharrison Aug 28, 2013
0861df5
change 'use' to 'require' so can handle in eval
Aug 28, 2013
f4daaab
Merge pull request #439 from teharrison/develop
teharrison Aug 28, 2013
562fbe9
eval'ed up imports to use api.cgi and resource.pm in kbase
Aug 29, 2013
3182144
Merge pull request #440 from teharrison/develop
teharrison Aug 29, 2013
37dee3a
m5nr info updates
Aug 29, 2013
cbf4d09
m5nr info updates
Aug 29, 2013
2560292
m5nr sql updates
Aug 29, 2013
ca3f4ab
Merge pull request #441 from teharrison/develop
teharrison Aug 29, 2013
79f934f
Merge branch 'develop' of github.com:MG-RAST/MG-RAST
Sep 4, 2013
d56e186
Edited Makefile to compile metazen code in parent directory if present
jaredbischof Sep 4, 2013
1c22257
Edited Makefile to compile metazen code
jaredbischof Sep 4, 2013
747adf7
Merge pull request #442 from jaredbischof/develop
jaredbischof Sep 4, 2013
62886b3
API: moved CV list to parent class
Sep 5, 2013
f139eaf
API: added hierarchal filtering to sequence download
Sep 5, 2013
a99898c
Merge branch 'develop' of git@github.com:MG-RAST/MG-RAST into develop
Sep 5, 2013
d62efe3
API: fix bug, better error checking
Sep 5, 2013
97234da
Merge pull request #443 from teharrison/develop
wilke Sep 5, 2013
57e9fa7
API: metagenome metadata option also outputs mixs data
Sep 6, 2013
7b69106
Merge branch 'develop' of git@github.com:MG-RAST/MG-RAST into develop
Sep 6, 2013
722e5dd
Merge pull request #444 from teharrison/develop
teharrison Sep 6, 2013
232f186
Merge branch 'develop' of github.com:MG-RAST/MG-RAST
Sep 9, 2013
a64a1c9
fixed data description of some list return types
Sep 9, 2013
0fe14e6
Merge pull request #436 from paczian/master
Sep 9, 2013
411b24b
API: fix param structure
Sep 9, 2013
19810f7
API: fix param structure
Sep 9, 2013
13823e4
API: added 'mixs' verbosity to mg instance
Sep 9, 2013
8292e8b
added metagenome stat keywords
Sep 9, 2013
8c1fd4f
added option for different tax level with alpha / rarefaction calcula…
Sep 9, 2013
d6d9185
typo fix
Sep 9, 2013
737fa35
API: added on-the-fly alpha diversity calculation to compute resource
Sep 9, 2013
b9787ce
API: added sequence stat keywords as metagenome search options
Sep 9, 2013
23c49cb
Merge pull request #445 from teharrison/develop
teharrison Sep 9, 2013
7c30640
info update
Sep 11, 2013
479b868
API: added pipeline option keywords, added function for using kbase i…
Sep 11, 2013
4ee9f7c
API: job resource for submission / id lookup
Sep 12, 2013
c310a85
split job->initilize into seperate 'reserve' and 'create' modes
Sep 12, 2013
d2f7ef4
API: added another hidden resource
Sep 12, 2013
6a9fe15
API: updated job reserve / create
Sep 12, 2013
58b3619
Merge branch 'develop' of git@github.com:MG-RAST/MG-RAST into develop
Sep 12, 2013
7bdbe9c
API: job resource bug fixes
Sep 12, 2013
ffe8cac
Merge pull request #446 from teharrison/develop
jaredbischof Sep 12, 2013
da57613
API: creating a KBase id for a new MG-RAST id is now optional
Sep 12, 2013
e8be171
Merge pull request #447 from teharrison/develop
teharrison Sep 12, 2013
b6f66b5
pixel pushing
Sep 13, 2013
a15f12f
Merge pull request #448 from paczian/master
Sep 13, 2013
5aff358
three more pixels...
Sep 13, 2013
bf09220
Merge pull request #449 from paczian/master
Sep 13, 2013
fde6e3a
API: fixed perl craziness with matrix filter
Sep 13, 2013
e33b3b7
added seed source src/Babel/info/ach_downloads src/Babel/info/ach_sou…
Sep 13, 2013
a580fb6
Merge pull request #450 from teharrison/develop
teharrison Sep 13, 2013
c5d1a7a
API: added missing option to post
Sep 13, 2013
d74c0bc
api: added otpion to reverse filter in matrix
Sep 13, 2013
e7eedf2
API use parent resource hierarchies
Sep 16, 2013
c95c7c0
API: show functional family ids
Sep 16, 2013
385e3ce
analysis module, more info to annotation-md5 return
Sep 17, 2013
cf16f0d
API: return annotation info for matrix/feature metadata
Sep 17, 2013
67453df
API: matrix resource now passes error message and status when ran in …
Sep 17, 2013
5db97c4
API: updated m5nr resource to use versions (1 or 9). default is 9.
Sep 17, 2013
38e7b3f
api: doc update
Sep 17, 2013
c242b16
m5tools, now use perl doc pod, update help message
Sep 17, 2013
b52aeeb
renamed: m5tools -> m5nr-tools
Sep 17, 2013
b029719
Merge branch 'develop' of git@github.com:MG-RAST/MG-RAST into develop
Sep 17, 2013
671dbab
Merge pull request #451 from teharrison/develop
teharrison Sep 17, 2013
c1d8759
api m5nr - added option for exact match with taxonomy and ontology fi…
Sep 17, 2013
e90c836
Merge pull request #452 from teharrison/develop
teharrison Sep 17, 2013
e985059
perl pod stuff fixed
Sep 17, 2013
3a60463
Merge pull request #453 from teharrison/develop
teharrison Sep 18, 2013
c51d172
API matrix: added option to not return feature metadata (can take a l…
Sep 18, 2013
8ff9e2b
Merge pull request #455 from teharrison/develop
teharrison Sep 18, 2013
1b57e9e
API m5nr, added option to search organism annotation via tax level
Sep 18, 2013
fb37ca4
Merge pull request #456 from teharrison/develop
teharrison Sep 18, 2013
94f8867
API - clean up download info
Sep 18, 2013
a975c4b
Merge pull request #457 from teharrison/develop
teharrison Sep 18, 2013
5838ebb
API download - hardcode file id for upload fastq / fasta
Sep 18, 2013
c573418
Merge pull request #458 from teharrison/develop
teharrison Sep 18, 2013
0090ff6
Fixed 'set preferences' for auth key generation on ManagePreferences …
jaredbischof Sep 19, 2013
0bc244e
Merge pull request #459 from jaredbischof/develop
jaredbischof Sep 19, 2013
47ee4e4
another perldoc fix
Sep 19, 2013
fa7c966
Merge pull request #460 from teharrison/develop
teharrison Sep 19, 2013
3c06d5f
another perldoc fix
Sep 19, 2013
66e1cf8
Merge pull request #461 from teharrison/develop
teharrison Sep 19, 2013
f168b28
API m5nr - added options to 1. return inverse query results for organ…
Sep 19, 2013
270498e
Merge pull request #462 from teharrison/develop
teharrison Sep 19, 2013
1e395b2
added m5nr-tools example
Sep 19, 2013
3ffd932
Merge pull request #463 from teharrison/develop
teharrison Sep 19, 2013
88b3de3
upload page, now accept small assembeled datasets
Sep 20, 2013
489a0cd
Merge pull request #464 from teharrison/develop
teharrison Sep 20, 2013
9610472
metadata template and data validation
Oct 2, 2013
5a1920f
Edit to make search from home page work
jaredbischof Oct 3, 2013
f310fc0
Merge pull request #465 from jaredbischof/develop
jaredbischof Oct 3, 2013
1943d46
Merge branch 'develop' of github.com:MG-RAST/MG-RAST
Oct 9, 2013
6285883
added shock node update and tag update
Oct 9, 2013
2dff17e
new template validation resource
Oct 9, 2013
75131db
api: added config varibale to change base api url
Oct 9, 2013
5aa4ba5
Merge pull request #467 from teharrison/develop
teharrison Oct 9, 2013
90a81f6
Merge pull request #466 from paczian/master
Oct 10, 2013
b818e55
valid kbase authentication w/o an mgrast user will not return an erro…
Oct 11, 2013
7172feb
Merge pull request #468 from paczian/master
Oct 11, 2013
e46c020
fixed login vulnerability
Oct 11, 2013
2c8af89
Merge pull request #469 from paczian/master
Oct 11, 2013
f7c8c9b
fixed error in kbase auth
Oct 14, 2013
1891e26
Merge pull request #470 from paczian/master
Oct 14, 2013
4f57f5b
fix perl warnings that spam logs
Oct 15, 2013
077efc2
Merge pull request #471 from teharrison/develop
teharrison Oct 15, 2013
3852758
fix bug with .fq files being renamed .fna
Oct 18, 2013
f4fbff6
Merge pull request #472 from teharrison/develop
teharrison Oct 18, 2013
6a8af1d
fix display of M5RNA in workbench
Oct 21, 2013
da25207
Merge pull request #473 from teharrison/develop
teharrison Oct 21, 2013
2c622ff
inputting metagenome id in home page search box redirects to that met…
Oct 21, 2013
eda74cb
Merge branch 'develop' of github.com:MG-RAST/MG-RAST into develop
Oct 21, 2013
6be3b48
API: display proper error message when trying to view metagenomes sti…
Oct 23, 2013
afddfd5
API: give error when invalid option combinations used
Oct 23, 2013
e4c6075
Merge pull request #474 from teharrison/develop
teharrison Oct 23, 2013
8227b8c
Merge branch 'master' of github.com:MG-RAST/MG-RAST
Nov 1, 2013
9bfb524
Merge branch 'develop' of github.com:MG-RAST/MG-RAST
Nov 1, 2013
0172424
Merge pull request #475 from paczian/master
Nov 7, 2013
e9fa77c
More explicit labeling of what join paired-ends operation does
jaredbischof Nov 12, 2013
34b2aac
Merge pull request #476 from jaredbischof/develop
jaredbischof Nov 12, 2013
5bc5c4b
restore abundance profile in new dir struct
teharrison Nov 20, 2013
d8791aa
api: added mixs compilent flag to mg object
Nov 20, 2013
d9a7670
Merge branch 'master' of git@github.com:teharrison/MG-RAST into develop
Nov 20, 2013
9b0959a
Merge pull request #477 from teharrison/develop
teharrison Nov 20, 2013
b441532
restore abundance profile - fix bug
teharrison Nov 20, 2013
d09d053
Merge branch 'develop' of github.com:MG-RAST/MG-RAST into develop
teharrison Nov 20, 2013
2e0b2fa
updated to work with mg-rast or kbase auth
Nov 20, 2013
bc472d2
Merge pull request #478 from paczian/master
teharrison Nov 20, 2013
32ab7d1
fixed auth bugs
Nov 20, 2013
2ca02fb
Merge pull request #479 from paczian/master
Nov 20, 2013
8e78526
Making Shock node for metadata template and mgrast shock auth config …
jaredbischof Nov 20, 2013
5b0e49b
Changed metadata template in Shock to be stored as a file so we don't…
jaredbischof Nov 21, 2013
23b5425
Merge pull request #480 from jaredbischof/develop
jaredbischof Nov 21, 2013
79333cf
Updates to get 'data' function of 'validation' resource working.
jaredbischof Nov 21, 2013
b7d911a
Changed update_shock_tags() code to use HTTP::Request::Common instead…
jaredbischof Nov 21, 2013
e1031ef
Updated information we are putting into attributes after validating a…
jaredbischof Nov 21, 2013
cc1ca76
Merge pull request #481 from jaredbischof/develop
jaredbischof Nov 21, 2013
d1b53e2
merged
Nov 21, 2013
2a8f414
fixed version
Nov 22, 2013
aeef1b2
Merge pull request #482 from paczian/master
Nov 22, 2013
dac2dc3
Adding example node ids
jaredbischof Nov 22, 2013
ab9569c
Merge pull request #483 from jaredbischof/develop
jaredbischof Nov 22, 2013
c00d8cb
better error msg
Nov 26, 2013
3fcb2c4
Merge branch 'develop' of github.com:MG-RAST/MG-RAST into develop
Nov 26, 2013
789fba6
Merge pull request #484 from teharrison/develop
teharrison Dec 5, 2013
7b4269c
fix biom format
Dec 5, 2013
962e099
Merge pull request #485 from teharrison/develop
teharrison Dec 5, 2013
4f2272b
Adding updated Metadata template
jaredbischof Dec 9, 2013
df665aa
Merge pull request #486 from jaredbischof/develop
jaredbischof Dec 9, 2013
cd658e1
ach download update
Dec 11, 2013
98f41aa
script for dumping SEED from CDMI
Dec 11, 2013
c2b1059
new R script for significance
Dec 11, 2013
8b484a4
api compute: update R calls, add significance test
Dec 11, 2013
6d15774
Merge branch 'develop' of github.com:MG-RAST/MG-RAST into develop
Dec 16, 2013
762825e
add alias to solr dump
Dec 16, 2013
35c26fc
changed job display options
Dec 17, 2013
f8badd4
hack for MO to work
Dec 18, 2013
014cdf9
fuzzy math update for wonky rna numbers
Dec 18, 2013
2f114bd
fix overview text
Dec 18, 2013
08f4456
Merge pull request #487 from teharrison/develop
teharrison Dec 18, 2013
2fc3000
Updated text for the file types that our unpack function accepts.
jaredbischof Dec 20, 2013
d1a0fb2
Merge branch 'develop' of https://github.com/MG-RAST/MG-RAST into dev…
jaredbischof Dec 20, 2013
9a543cc
Merge pull request #488 from jaredbischof/develop
jaredbischof Dec 20, 2013
7585fa9
m5nr download updates
Dec 20, 2013
dca7fc8
api: add alias to m5nr
Dec 20, 2013
41065fd
Edited search on home page to link directly to project pages.
jaredbischof Dec 20, 2013
dd5e585
Fix to regex
jaredbischof Dec 20, 2013
f299c29
Merge pull request #489 from jaredbischof/develop
jaredbischof Dec 20, 2013
a4e90f5
bug fixes
Jan 7, 2014
d718950
Merge branch 'develop' of github.com:MG-RAST/MG-RAST into develop
Jan 16, 2014
2511692
m5nr change default
Jan 16, 2014
ceaf412
Merge pull request #490 from teharrison/develop
teharrison Jan 16, 2014
acd4067
Merge pull request #1 from MG-RAST/develop
Jan 20, 2014
4982866
Change user count and link to tech report
Jan 20, 2014
edac640
Merge pull request #491 from mdsouza/develop
Jan 20, 2014
43024c1
Update preprocessing.r
DrOppenheimer Jan 21, 2014
d8900df
Bug in solr query for users that are logged in and dont have star rig…
jaredbischof Jan 22, 2014
6d6adbf
Merge pull request #492 from jaredbischof/develop
jaredbischof Jan 22, 2014
2fb317a
applied parser fix
Jan 22, 2014
435ac5e
Merge branch 'develop' of github.com:MG-RAST/MG-RAST
Jan 22, 2014
0da11a0
Merge pull request #493 from paczian/master
Jan 22, 2014
d5b6963
default timeout set to 7 days
Feb 18, 2014
48563aa
fixed cleanup in deny user request
Feb 18, 2014
fcc94da
fixed quoting error in barcharts
Feb 18, 2014
cf8dd30
added double quote fix to previous fix
Feb 18, 2014
3b1bb2a
fixed clickable abundace cells in workbench table
Feb 18, 2014
d8640a1
Merge pull request #494 from paczian/master
Feb 19, 2014
6f43043
fixed missing params from last update
Feb 19, 2014
1747ea2
Merge pull request #495 from paczian/master
Feb 19, 2014
9e1049c
add pipeline options to output
Feb 19, 2014
0992c8a
Merge branch 'develop' of github.com:MG-RAST/MG-RAST into develop
Feb 19, 2014
405c0f0
fix to pipeline options output
Feb 19, 2014
304febf
Merge pull request #496 from teharrison/develop
teharrison Feb 19, 2014
58be151
api: bug fix to metagenome paramater output
Feb 20, 2014
136fd1b
Merge pull request #497 from teharrison/develop
teharrison Feb 20, 2014
b66a6dc
fix divide by zero bug
Feb 21, 2014
b0ad64a
Merge pull request #498 from teharrison/develop
teharrison Feb 21, 2014
7304524
api: add required id to doc
Feb 21, 2014
21be65d
Merge pull request #499 from teharrison/develop
teharrison Feb 21, 2014
c6b484e
fix overview piechart negative values
Feb 21, 2014
126e817
Merge pull request #500 from teharrison/develop
teharrison Feb 21, 2014
8942bc7
api: fix response for unknown id
Feb 21, 2014
340fe69
Merge pull request #501 from teharrison/develop
teharrison Feb 21, 2014
f66746b
defaulting site_name to Metagenomics
Feb 24, 2014
2336abe
Merge pull request #502 from paczian/master
Feb 24, 2014
2b1cbb0
Merge pull request #2 from MG-RAST/develop
Feb 24, 2014
7981b9b
change manual name and link
Feb 24, 2014
ace4895
Merge pull request #503 from mdsouza/develop
Feb 24, 2014
d2d3290
Example invokations had invalid JSON
Feb 25, 2014
cf24cb6
Merge remote-tracking branch 'mgrast/develop' into brokenexamples
Feb 25, 2014
1b3722b
Merge pull request #504 from wltrimbl/brokenexamples
teharrison Feb 25, 2014
a78a983
Re-ordered the table creation in the schema file so references work c…
danielolson5 Feb 27, 2014
145757f
Adding this where clause makes calls to metagenome resource with verb…
jaredbischof Mar 3, 2014
5390ad0
Merge branch 'develop' of https://github.com/MG-RAST/MG-RAST into dev…
jaredbischof Mar 3, 2014
15986c6
Merge pull request #507 from jaredbischof/develop
jaredbischof Mar 3, 2014
f681174
Returning error msg when user tries to decompress unsupported file co…
jaredbischof Mar 4, 2014
da03787
Merge branch 'develop' of https://github.com/MG-RAST/MG-RAST into dev…
jaredbischof Mar 4, 2014
9cb1fa4
Merge pull request #508 from jaredbischof/develop
jaredbischof Mar 4, 2014
b3b5eeb
update MetaDataCV schema
Mar 7, 2014
284697a
update metadata module and api resource to use locally hosted ontolog…
Mar 7, 2014
3e69c7a
fix metadata template stuff
Mar 10, 2014
6e50221
api annotation resource: added option to filter by md5, single with G…
Mar 11, 2014
860f830
Merge branch 'develop' of git@github.com:MG-RAST/MG-RAST into develop
Mar 11, 2014
4d60c88
change table names from test to prod
Mar 12, 2014
8534bfa
annotation API, fix to md5 POST
Mar 12, 2014
bcb07c9
Merge pull request #509 from teharrison/develop
teharrison Mar 12, 2014
d536943
Merge pull request #505 from danielolson5/develop
teharrison Mar 12, 2014
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ OVERRIDES = \
# if you change all: make sure all: in standard.mk is consistent.
# api-doc target must be run last.

all: installdirs $(TOOL_HDR) lib
all: installdirs $(TOOL_HDR) lib metazen

# Use the PACKAGES macro to transform lib, bin, etc., targets
# into package level dependencies, e.g., lib -> PkgA.lib, PkgB.lib
Expand Down Expand Up @@ -87,6 +87,13 @@ installdirs: $(libdir) $(bindir) $(cgidir) $(cgidir)/Html $(cgidir)/Html/css $(t
- mkdir -p $(bindir)
- mkdir -p $(cgidir)

metazen:
if [ -d "../metazen" ]; then \
cd ../metazen; \
make; \
cd -; \
fi

# the subdirs are for putting python pyc etc. into

$(libdir): $(foreach var,$(PACKAGES),$(libdir)/$(var).installdirs)
Expand Down
22 changes: 17 additions & 5 deletions bin/api2html.pl
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,24 @@ sub usage {

# read in parameters
my $url = '';
my $site_name = '';
my $site_name = 'Metagenomics';
my $outfile = '';
my $help = 0;

GetOptions ( 'url=s' => \$url,
'site_name=s' => \$site_name,
'outfile=s' => \$outfile );
'outfile=s' => \$outfile,
'help!' => \$help );

unless ($url and $site_name and $outfile) {
&usage();
exit 0;
if ($help) {
&usage();
exit 0;
}

unless ($url and $outfile) {
print "missing required paramater\n";
&usage();
exit 1;
}

# initialize json object and user agent
Expand Down Expand Up @@ -104,7 +112,11 @@ sub usage {
$html .= ")</li><p>";
$html .= "This parameter value can be chosen from the following (the first being default):</p><ul style='list-style: none;'>";
foreach my $cvitem (@{$pm->[1]}) {
if (ref($cvitem)) {
$html .= "<li><b>".$cvitem->[0]."</b> - ".$cvitem->[1]."</li>";
} else {
$html .= "<li><b>".$cvitem."</b></li>";
}
}
$html .= "</ul><br>";
}
Expand Down
7 changes: 4 additions & 3 deletions src/Babel/bin/create_pg_indexes.sql
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ CREATE INDEX sources_name ON sources (name);
CREATE INDEX sources_type ON sources (type);

# load ontologies table
INSERT INTO ontologies (level1,level2,level3,level4,id,type) SELECT level1,level2,level3,level4,id,'KO' FROM ontology_kegg;
INSERT INTO ontologies (level1,level2,level3,level4,id,type) SELECT level1,level2,level3,level4,id,'Subsystems' FROM ontology_seed;
INSERT INTO ontologies (level1,level2,level3,id,type) SELECT level1,level2,level3,id,type FROM ontology_eggnog;
INSERT INTO ontologies (level1,level2,level3,level4,id,source) SELECT level1,level2,level3,level4,id,12 FROM ontology_kegg;
INSERT INTO ontologies (level1,level2,level3,level4,id,source) SELECT level1,level2,level3,level4,id,14 FROM ontology_seed;
INSERT INTO ontologies (level1,level2,level3,id,source) SELECT level1,level2,level3,id,10 FROM ontology_eggnog WHERE type='COG';
INSERT INTO ontologies (level1,level2,level3,id,source) SELECT level1,level2,level3,id,13 FROM ontology_eggnog WHERE type='NOG';

54 changes: 28 additions & 26 deletions src/Babel/bin/download_ach_sources.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ fi

# Config
DOWNLOAD_DIR=$1
SOURCES="eggNOG FungalDB Greengenes IMG InterPro KEGG KO NR PATRIC Phantome RefSeq RDP SEED SILVA UniProt"
SOURCES="eggNOG FungiDB Greengenes IMG InterPro KEGG MO NR PATRIC Phantome RefSeq RDP SEED SILVA UniProt"

echo Starting Download for ACH `date`

Expand All @@ -24,63 +24,65 @@ do
done

echo Downloading eggNOG `date`
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG http://eggnog.embl.de/version_3.0/data/downloads/fun.txt.gz
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG http://eggnog.embl.de/version_3.0/data/downloads/UniProtAC2eggNOG.3.0.tsv.gz
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG http://eggnog.embl.de/version_3.0/data/downloads/COG.funccat.txt.gz
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG http://eggnog.embl.de/version_3.0/data/downloads/NOG.funccat.txt.gz
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG http://eggnog.embl.de/version_3.0/data/downloads/COG.description.txt.gz
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG http://eggnog.embl.de/version_3.0/data/downloads/NOG.description.txt.gz
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG 'http://eggnog.embl.de/version_3.0/data/downloads/fun.txt.gz'
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG 'http://eggnog.embl.de/version_3.0/data/downloads/UniProtAC2eggNOG.3.0.tsv.gz'
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG 'http://eggnog.embl.de/version_3.0/data/downloads/COG.funccat.txt.gz'
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG 'http://eggnog.embl.de/version_3.0/data/downloads/NOG.funccat.txt.gz'
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG 'http://eggnog.embl.de/version_3.0/data/downloads/COG.description.txt.gz'
wget -v -N -P ${DOWNLOAD_DIR}/eggNOG 'http://eggnog.embl.de/version_3.0/data/downloads/NOG.description.txt.gz'

echo Downloading FungalDB `date`
wget -v -N -P ${DOWNLOAD_DIR}/FungalDB http://fungalgenomes.org/public/mobedac/for_VAMPS/fungalITSdatabaseID.taxonomy.seqs.gz

#echo Downloading GO hierarchy `date`
#time lftp -c "open -e 'mirror -v --no-recursion -I gene_ontology.1_2.obo /pub/go/ontology/obo_format_1_2/ ${DOWNLOAD_DIR}/GO' ftp://ftp.geneontology.org"
echo Downloading FungiDB `date`
wget -v -N -P ${DOWNLOAD_DIR}/FungalDB 'http://fungalgenomes.org/public/mobedac/for_VAMPS/fungalITSdatabaseID.taxonomy.seqs.gz'

echo Downloading Greengenes `date`
wget -v -N -P ${DOWNLOAD_DIR}/Greengenes http://greengenes.lbl.gov/Download/Sequence_Data/Fasta_data_files/current_GREENGENES_gg16S_unaligned.fasta.gz
wget -v -N -P ${DOWNLOAD_DIR}/Greengenes 'http://greengenes.lbl.gov/Download/Sequence_Data/Fasta_data_files/current_GREENGENES_gg16S_unaligned.fasta.gz'

echo Downloading IMG `date`
time lftp -c "open -e 'mirror -v --no-recursion -I img_core_v400.tar /pub/IMG/ ${DOWNLOAD_DIR}/IMG' ftp://ftp.jgi-psf.org"
echo ftp path is missing
#time lftp -c "open -e 'mirror -v --no-recursion -I img_core_v400.tar /pub/IMG/ ${DOWNLOAD_DIR}/IMG' ftp://ftp.jgi-psf.org"

echo Downloading InterPro `date`
time lftp -c "open -e 'mirror -v --no-recursion -I names.dat /pub/databases/interpro/ ${DOWNLOAD_DIR}/InterPro' ftp://ftp.ebi.ac.uk"

echo Download KEGG `date`
# ftp is no longer accessable
echo ftp is no longer accessable
#time lftp -c "open -e 'mirror -v --no-recursion -I genome /pub/kegg/genes/ ${DOWNLOAD_DIR}/KEGG' ftp://ftp.genome.ad.jp"
#time lftp -c "open -e 'mirror -v --no-recursion -I genes.tar.gz /pub/kegg/release/current/ ${DOWNLOAD_DIR}/KEGG' ftp://ftp.genome.ad.jp"

echo Download KO `date`
# ftp is no longer accessable
#time lftp -c "open -e 'mirror -v --no-recursion -I ko /pub/kegg/genes/ ${DOWNLOAD_DIR}/KO' ftp://ftp.genome.ad.jp"
#time lftp -c "open -e 'mirror -v --parallel=2 -I *.keg /pub/kegg/brite/ko/ ${DOWNLOAD_DIR}/KO' ftp://ftp.genome.ad.jp"

echo Download MO `date`
# issue with recursive wget and links on page, this hack works
for i in `seq 1 907348`; do wget -N -P ${DOWNLOAD_DIR}/MO http://www.microbesonline.org/genbank/${i}.gbk.gz 2> /dev/null; done
wget -v -N -P ${DOWNLOAD_DIR}/MO http://www.microbesonline.org/genbank/10000550.gbk.gz

echo Downloading NCBI NR `date`
time lftp -c "open -e 'mirror -v -e --no-recursion -I nr.gz /blast/db/FASTA/ ${DOWNLOAD_DIR}/NR' ftp://ftp.ncbi.nih.gov"

echo Downloading PATRIC `date`
time lftp -c "open -e 'mirror -v --parallel=2 -I *.PATRIC.gbf /patric2/genomes/ ${DOWNLOAD_DIR}/PATRIC' http://brcdownloads.vbi.vt.edu"

echo Downloading Phantome `date`
wget -v -N -P ${DOWNLOAD_DIR}/Phantome http://www.phantome.org/Downloads/proteins/all_sequences/phage_proteins_1375354802.fasta.gz
wget -v -O ${DOWNLOAD_DIR}/Phantome/phage_proteins.fasta.gz 'http://www.phantome.org/Downloads/proteins/all_sequences/current'

echo Downloading RefSeq `date`
echo Downloading NCBI RefSeq `date`
time lftp -c "open -e 'mirror -v -e --delete-first -I *.genomic.gbff.gz /refseq/release/complete/ ${DOWNLOAD_DIR}/RefSeq' ftp://ftp.ncbi.nih.gov"

echo Downloading RDP `date`
wget -v -N -P ${DOWNLOAD_DIR}/RDP http://rdp.cme.msu.edu/download/release10_32_unaligned.gb.gz
wget -v -N -P ${DOWNLOAD_DIR}/RDP 'http://rdp.cme.msu.edu/download/release11_1_Bacteria_unaligned.gb.gz'
wget -v -N -P ${DOWNLOAD_DIR}/RDP 'http://rdp.cme.msu.edu/download/release11_1_Archaea_unaligned.gb.gz'
wget -v -N -P ${DOWNLOAD_DIR}/RDP 'http://rdp.cme.msu.edu/download/release11_1_Fungi_unaligned.gb.gz'

echo Downloading SEED `date`
# can not fp, must extract painfully through SEED API
echo can not ftp, must extract painfully through SEED API
#time lftp -c "open -e 'mirror -v --no-recursion -I SEED.fasta /misc/Data/idmapping/ ${DOWNLOAD_DIR}/SEED' ftp://ftp.theseed.org"
#time lftp -c "open -e 'mirror -v --no-recursion -I subsystems2role.gz /subsystems/ ${DOWNLOAD_DIR}/SEED' ftp://ftp.theseed.org"

echo Downloading SILVA `date`
wget -v -N -P ${DOWNLOAD_DIR}/SILVA http://www.arb-silva.de/no_cache/download/archive/release_108/Exports/lsu-parc.fasta.tgz
wget -v -N -P ${DOWNLOAD_DIR}/SILVA http://www.arb-silva.de/no_cache/download/archive/release_108/Exports/lsu-parc.rast.tgz
wget -v -N -P ${DOWNLOAD_DIR}/SILVA http://www.arb-silva.de/no_cache/download/archive/release_108/Exports/ssu-parc.fasta.tgz
wget -v -N -P ${DOWNLOAD_DIR}/SILVA http://www.arb-silva.de/no_cache/download/archive/release_108/Exports/ssu-parc.rast.tgz
wget -v -N -P ${DOWNLOAD_DIR}/SILVA 'http://www.arb-silva.de/fileadmin/silva_databases/release_108/Exports/lsu-parc.fasta.tgz'
wget -v -N -P ${DOWNLOAD_DIR}/SILVA 'http://www.arb-silva.de/fileadmin/silva_databases/release_108/Exports/lsu-parc.rast.tgz'
wget -v -N -P ${DOWNLOAD_DIR}/SILVA 'http://www.arb-silva.de/fileadmin/silva_databases/release_108/Exports/ssu-parc.fasta.tgz'
wget -v -N -P ${DOWNLOAD_DIR}/SILVA 'http://www.arb-silva.de/fileadmin/silva_databases/release_108/Exports/ssu-parc.rast.tgz'

echo Downloading Uniprot `date`
time lftp -c "open -e 'mirror -v -e --delete-first -I uniprot_sprot.dat.gz /pub/databases/uniprot/current_release/knowledgebase/complete ${DOWNLOAD_DIR}/UniProt' ftp.uniprot.org"
Expand Down
89 changes: 50 additions & 39 deletions src/Babel/bin/dump_annotation_for_solr.pl
Original file line number Diff line number Diff line change
Expand Up @@ -30,70 +30,83 @@
print STDERR $usage; exit 1;
}
my $sth;
my $count;

if (! $output) {
print STDERR $usage; exit;
print STDERR $usage; exit 1;
}

my $dbh = DBI->connect("DBI:Pg:dbname=$dbname;host=$dbhost", $dbuser, '', {AutoCommit => 0});
unless ($dbh) { print STDERR "Error: " . $DBI::errstr . "\n"; exit 1; }

## skipping GO
print STDERR "Loading sources ...\n";
my $src_data = ['source_id','source','organization','description','type','url','email','link','title','version','download_date'];
my $src_sql = "SELECT _id AS source_id, name AS source, source AS organization, description, type, url, email, link, title, version, download_date FROM sources WHERE _id != 11";
my $src_lookup = $dbh->selectall_hashref($src_sql , "source_id");

print STDERR "Dumping sources ...\n";
open(SRC, ">$output.source") or die "Couldn't open $output.source for writing.\n";
print SRC to_json(lookup2data($src_lookup, $src_data, 'static_source'), {ascii => 1})."\n";
foreach my $set (values %$src_lookup) {
$set->{id} = 's_'.$set->{source_id};
$set->{object} = 'source';
print SRC to_json($set, {ascii => 1})."\n";
delete $set->{id};
delete $set->{object};
}
close(SRC);

print STDERR "Loading ontologies ...\n";
my $ont_sql = "SELECT id, level1, level2, level3, level4, source FROM ontologies";
my $ont_lookup = $dbh->selectall_hashref($ont_sql , "id");
foreach my $onts (grep {$_->{type} eq 'ontology'} values %$src_lookup) {
my $ont_data = ['id','level1','level2','level3','level4'];
if ($onts->{source} =~ /^[CN]OG$/) {
pop @$ont_data;
my $ont_sql = "SELECT o._id AS ontology_id, o.id AS accession, o.level1, o.level2, o.level3, o.level4, s.name AS source FROM ontologies o, sources s WHERE o.source = s._id AND s._id != 11";
my $ont_lookup = $dbh->selectall_hashref($ont_sql , "accession");

print STDERR "Dumping ontologies ...\n";
open(ONT, ">$output.ontology") or die "Couldn't open $output.ontology for writing.\n";
$count = 1;
foreach my $set (values %$ont_lookup) {
$set->{id} = 'o_'.$count;
$set->{object} = 'ontology';
if ($set->{source} =~ /^[NC]OG$/) {
delete $set->{level4};
}
my %sub_ont_lookup = map {$_, $ont_lookup->{$_}} grep {$ont_lookup->{$_}{source} eq $onts->{source_id}} keys %$ont_lookup;
print STDERR "Dumping ontology ".$onts->{source}." ...\n";
open(TAX, ">$output.".$onts->{source}) or die "Couldn't open $output.".$onts->{source}." for writing.\n";
print TAX to_json(lookup2data(\%sub_ont_lookup, $ont_data, 'static_'.$onts->{source}), {ascii => 1})."\n";
close(TAX);
print ONT to_json($set, {ascii => 1})."\n";
delete $set->{id};
delete $set->{object};
$count += 1;
}
close(ONT);

print STDERR "Loading organisms ...\n";
my $org_data = ['organism_id','organism','domain','phylum','class','order','family','genus','species','ncbi_tax_id'];
my $org_sql = "SELECT _id AS organism_id, name AS organism, tax_domain AS domain, tax_phylum AS phylum, tax_class AS class, tax_order AS order, tax_family AS family, tax_genus AS genus, tax_species AS species, ncbi_tax_id FROM organisms_ncbi";
my $org_lookup = $dbh->selectall_hashref($org_sql , "organism_id");

print STDERR "Dumping taxonomy ...\n";
open(TAX, ">$output.taxonomy") or die "Couldn't open $output.taxonomy for writing.\n";
print TAX to_json(lookup2data($org_lookup, $org_data, 'static_taxonomy'), {ascii => 1})."\n";
foreach my $set (values %$org_lookup) {
$set->{id} = 't_'.$set->{organism_id};
$set->{object} = 'taxonomy';
print TAX to_json($set, {ascii => 1})."\n";
delete $set->{id};
delete $set->{object};
}
close(TAX);

$dbh->disconnect;
exit;

print STDERR "Loading functions ...\n";
my $func_sql = "SELECT _id AS function_id, name AS function FROM functions";
my $func_lookup = $dbh->selectall_hashref($func_sql , "function_id");

print STDERR "Loading md5s ...\n";
my $md5_sql = "SELECT _id AS md5_id, md5 FROM md5s";
my $md5_lookup = $dbh->selectall_hashref($md5_sql , "md5_id");
my $md5_lookup = $dbh->selectall_hashref($md5_sql , "md5");

open(DUMP, ">$output") or die "Couldn't open $output for writing.\n";
open(DUMP, ">$output.annotation") or die "Couldn't open $output.annotation for writing.\n";
# rna
print STDERR "Dumping rna annotations ...\n";
my $count = 1;
$count = 1;
$sth = $dbh->prepare("SELECT md5, id, function, organism, source FROM md5_rna WHERE tax_rank = 1");
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
my ($md5, $id, $func, $org, $src) = @row;
my $data = { 'id' => $count, 'accession' => $id };
my $data = { 'id' => 'a_'.$count, 'accession' => $id, 'object' => 'annotation' };
if ($md5 && exists($md5_lookup->{$md5})) {
map { $data->{$_} = $md5_lookup->{$md5}{$_} } keys %{$md5_lookup->{$md5}};
}
Expand All @@ -117,7 +130,18 @@
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
my ($md5, $id, $func, $org, $src) = @row;
my $data = { 'id' => $count, 'accession' => $id };
my $aliases = $dbh->selectall_arrayref("SELECT alias_source, alias_id FROM aliases_protein WHERE id=".$dbh->quote($id));
my $data = { 'id' => 'a_'.$count, 'accession' => $id, 'object' => 'annotation' };
if ($aliases && (@$aliases > 0)) {
$data->{alias} = [];
foreach my $aset (@$aliases) {
my ($asrc, $aid) = @$aset;
if ($asrc && $aid && ($aid !~ /^$asrc:/)) {
$aid = $asrc.":".$aid;
}
push @{$data->{alias}}, $aid;
}
}
if ($md5 && exists($md5_lookup->{$md5})) {
map { $data->{$_} = $md5_lookup->{$md5}{$_} } keys %{$md5_lookup->{$md5}};
}
Expand All @@ -141,15 +165,15 @@
$sth->execute() or die "Couldn't execute statement: " . $sth->errstr;
while (my @row = $sth->fetchrow_array()) {
my ($md5, $id, $func, $src) = @row;
my $data = { 'id' => $count, 'accession' => $id };
my $data = { 'id' => 'a_'.$count, 'accession' => $id, 'object' => 'annotation' };
if ($md5 && exists($md5_lookup->{$md5})) {
map { $data->{$_} = $md5_lookup->{$md5}{$_} } keys %{$md5_lookup->{$md5}};
}
if ($func && exists($func_lookup->{$func})) {
map { $data->{$_} = $func_lookup->{$func}{$_} } keys %{$func_lookup->{$func}};
}
if ($id && exists($ont_lookup->{$id})) {
map { $data->{$_} = $ont_lookup->{$id}{$_} } grep { exists $ont_lookup->{$id}{$_} } ('level1', 'level2', 'level3', 'level4');
map { $data->{$_} = $ont_lookup->{$id}{$_} } keys %{$ont_lookup->{$id}};
}
if ($src && exists($src_lookup->{$src})) {
map { $data->{$_} = $src_lookup->{$src}{$_} } grep { exists $src_lookup->{$src}{$_} } ('source_id', 'source', 'type');
Expand All @@ -160,16 +184,3 @@
$sth->finish;
close(DUMP);
$dbh->disconnect;

sub lookup2data {
my ($lookup, $list, $id) = @_;
my %data = map {$_, []} @$list;
foreach my $set (values %$lookup) {
foreach my $field (@$list) {
my $value = exists($set->{$field}) ? $set->{$field} : undef;
push @{$data{$field}}, $value;
}
}
$data{id} = $id;
return \%data;
}
15 changes: 7 additions & 8 deletions src/Babel/bin/load_md5protein2db.pl
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,13 @@
chomp $line;
my ($id, @aliases) = split(/\t/, $line);
foreach (@aliases) {
if ($_ =~ /^(\S+?):(\S+)$/) {
print ATMP "$id\t$2\t$1\n";

$count += 1;
unless ($count % 1000000) {
if ($verbose) { print STDERR "$count:\t$id , $2 , $1\n"; }
}
}
if ($_ =~ /^(\S+?):(\S+)$/) {
print ATMP "$id\t$2\t$1\n";
$count += 1;
unless ($count % 1000000) {
if ($verbose) { print STDERR "$count:\t$id , $2 , $1\n"; }
}
}
}
}
close AFILE;
Expand Down
Loading