Repository navigation
Expand file tree
/
Copy pathpipeline.py
More file actions
499 lines (447 loc) · 30 KB
/
Copy pathpipeline.py
File metadata and controls
499 lines (447 loc) · 30 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
import argparse
import os
import sys
from tasks_v2 import Supervisor, Task
import task_functions_v2 as tf
from assembler import gen_assembly_supervisor
from quality import gen_quality_supervisor
from annotater import gen_annotation_supervisor
from expression import gen_expression_supervisor
from itertools import chain
import time
from manage_database import main as db_install
#####################____Argument_Parsers____#####################
def master_args():
parser = argparse.ArgumentParser(description=('Description: MMT is a powerful convenience tool that '
'allows a user to run a full transcriptomics pipeline with a single command. MMT will manage the '
'assembly of the reads, annotate the resultant transcripts, and perform a differential '
'expression analysis of your dataset. In addition to performing the above three phases (assembly, '
'annotation, expression) in a single command, each of these three principle phases can be run '
'individually provided appropriate input.') )
parser.add_argument('tool_selector',help=('Specifies what portions of the transcriptomics '
'pipeline should be run. Appropriate values are "full", "assembly", "annotation", or "expression". '
'To see the arguments available for each of these tools, simply use the tool argument followed by '
'"-h". For example. To see all arguments available to the "assembly" tool, use "assembly -h" as '
'arguments') )
parser.add_argument('place_holder',nargs=argparse.REMAINDER,help=argparse.SUPPRESS)
return parser.parse_args()
def full_args():
parser = argparse.ArgumentParser(description="Selected_tool : Full. Executing this tool will run the entire transcriptomics pipeline. This tool requires a specially formatted CSV file to describe the input. Please see the online documentaiton to learn how to format these files.")
parser.add_argument('tool_selector',help=argparse.SUPPRESS)
parser.add_argument('--csv', help='A CSV file specifying fastq input, basenames for DE, and factors for DE. See online documentation for details on formating the CSV file.',default=None)
parser.add_argument('-test',help='Use this flag to test the pipeline.',action='store_true')
parser.add_argument('-no_rmdup',help='Use thie flag to disable the removing duplicates portion of the pre-assembly read cleaning.',action='store_true')
parser.add_argument('-no_trim',help='Use this flag to disable all trimming portions of pre-assembly read cleaning. Duplicate and low quality reads will not be removed. Subsampling will still be executed.',action='store_true')
parser.add_argument('-trimmomatic',help='Use trimmomatic instead of prinseq to trime reads',action='store_true')
parser.add_argument('--subsample_size',help='If greater than this number of reads (in millions) is provided, sub sample down to this number. Use 0 to signal that no subsampling should be performed. The deafult value is 50.', default=50,type=float)
parser.add_argument('--subsample_seed',help='A seed used to initialize the random number generator used during random sampling.')
parser.add_argument('--truncate',help='snip reads down to this size if longer than this size. Default is no truncations.',type=int,default=-1)
parser.add_argument('-trinity_normalization',action='store_true',help='Use this flag to use the trinity normalization option')
# parser.add_argument('-a','--assembly',help='A fasta transcriptome assembly that will be used for computing expression levels.')
parser.add_argument('-rnaspades',help='Use this flag to specify that assembly should be performed by rnaSPAdes rather than the default Trinity.',action='store_true')
parser.add_argument('-cegma',help='Use this flag to run cegma as part of the annotation pipeline. Cegma is an old tool for assesing the quality of assemblies. Normal behavior of the pipeline is to use busco for assesing assemblies. Using this flag will run cegma in addition to Busco.',action='store_true')
parser.add_argument('-blastplus',action='store_true',help='Use the blast+ tool suite instead of diamond to align your transcripts to the references.')
parser.add_argument('-uniref90',help='Use this flag to enable the uniref-90 blast runs as part of the annotation pipeline.',action='store_true')
parser.add_argument('-nr',help='Use this flag to enable the NR (non-redundant protein database) blast runs as part of the annotation pipeline. This could take a very long time to complete.',action='store_true')
parser.add_argument('-signalp',action='store_true',help='Use this flag to execute signalP during annotation. Only use if you have installed signalP.')
parser.add_argument('-tmhmm',action='store_true',help='Use this flag to execute tmhmm during annotation. Only use if you have installed tmhmm.')
parser.add_argument('-rnammer',action='store_true',help='Use this flag to execute rnammer during annotation. Only use if you have installed rnammer.')
parser.add_argument('--busco_ref',help='Set the reference that busco will use for analysis',default='metazoa')
parser.add_argument('--transrate_ref',help='A reference that transrate will use to evaluate the quality of your assembly.',default='')
parser.add_argument('--model', help='An optional list of space seperated values used to run differential expression. This is particularly useful for refining Differential Expression runs as it allow you to use the same input CSV file and perform new comparisons.')
parser.add_argument('-no_log',help='Pipeline will delete log files.',action='store_true')
parser.add_argument('-force',help='Use this flag to perform a fresh run of the pipeline. All steps will be executed regradless of what has already been performed.',action='store_true')
parser.add_argument('--cpu', help='Sets the process cap for execution. Default is 12. Use 0 to indicate no process cap should be used.',default=12,type=int)
parser.add_argument('--email',help='Pipeline will send emails informing you of runstate.')
parser.add_argument('--out_dir', help='Path to the ouput location. Defaults to assemblies directory inside pipeline',default=tf.PATH_ASSEMBLIES)
parser.add_argument('-o','--out_name', help='The name of the output directory to be made in out_dir. If unused, name will be inherited from input file names')
return parser.parse_args()
def assembly_args():
parser = argparse.ArgumentParser(description='Selected_tool : Assembler. Executing this tool will clean all provided reads and assemble them.')
parser.add_argument('tool_selector',help=argparse.SUPPRESS)
parser.add_argument('-test',help='Use this flag to test the pipeline.',action='store_true')
parser.add_argument('-u','--unpaired',help='A comma seperated list of unpaired fastq files.')
parser.add_argument('-1','--fastq1',help='A comma seperated list of fastq files. Each file should be paired with the same indexed file in fastq2.')
parser.add_argument('-2','--fastq2',help='A comma seperated list of fastq files. Each file should be paired with the same indexed file in fastq1.')
parser.add_argument('-rnaspades',help='Use this flag to specify that assembly should be performed by rnaSPAdes rather than the default Trinity.',action='store_true')
parser.add_argument('-trinity_normalization',action='store_true',help='Use this flag to use the trinity normalization option')
parser.add_argument('-no_rmdup',help='Use thie flag to disable the removing duplicates portion of the pre-assembly read cleaning.',action='store_true')
parser.add_argument('-no_trim',help='Use this flag to disable all trimming portions of pre-assembly read cleaning. Duplicate and low quality reads will not be removed. Subsampling will still be executed.',action='store_true')
parser.add_argument('-trimmomatic',help='Use trimmomatic instead of prinseq to trime reads',action='store_true')
parser.add_argument('--subsample_size',help='If greater than this number of reads (in millions) is provided, sub sample down to this number. Use 0 to signal that no subsampling should be performed. The deafult value is 50.', default=50,type=float)
parser.add_argument('--subsample_seed',help='A seed used to initialize the random number generator used during random sampling.')
parser.add_argument('--truncate',help='snip reads down to this size if longer than this size. Default is no truncations.',type=int,default=-1)
#parser.add_argument('-cegma',help='Use this flag to run cegma as part of the annotation pipeline. Cegma is an old tool for assesing the quality of assemblies. Normal behavior of the pipeline is to use busco for assesing assemblies. Using this flag will run cegma in addition to Busco.',action='store_true')
#parser.add_argument('--busco_ref',help='Set the reference that busco will use for analysis',default='metazoa')
#parser.add_argument('--transrate_ref',help='A reference that transrate will use to evaluate the quality of your assembly.',default='')
parser.add_argument('-no_log',help='Pipeline will delete log files.',action='store_true')
parser.add_argument('-force',help='Use this flag to perform a fresh run of the pipeline. All steps will be executed regradless of what has already been performed.',action='store_true')
parser.add_argument('--cpu', help='Sets the process cap for execution. Default is 12. Use 0 to indicate no process cap should be used.',default=12,type=int)
parser.add_argument('--email',help='Pipeline will send emails informing you of runstate.')
parser.add_argument('--out_dir', help='Path to the ouput location. Defaults to assemblies directory inside pipeline',default=tf.PATH_ASSEMBLIES)
parser.add_argument('-o','--out_name', help='The name of the output directory to be made in out_dir. If unused, name will be inherited from input file names')
return parser.parse_args()
def quality_args():
parser = argparse.ArgumentParser(description='Selected_tool : Quality. Executing this tool will assess a transcriptome assembly.')
parser.add_argument('tool_selector',help=argparse.SUPPRESS)
parser.add_argument('-test',help='Use this flag to test the pipeline.',action='store_true')
parser.add_argument('-u','--unpaired',help='A comma seperated list of unpaired fastq files.')
parser.add_argument('-1','--fastq1',help='A comma seperated list of fastq files. Each file should be paired with the same indexed file in fastq2.')
parser.add_argument('-2','--fastq2',help='A comma seperated list of fastq files. Each file should be paired with the same indexed file in fastq1.')
parser.add_argument('-a','--assembly',help='A fasta transcriptome assembly that will be used for computing expression levels.')
parser.add_argument('-cegma',help='Use this flag to run cegma as part of the annotation pipeline. Cegma is an old tool for assesing the quality of assemblies. Normal behavior of the pipeline is to use busco for assesing assemblies. Using this flag will run cegma in addition to Busco.',action='store_true')
parser.add_argument('--busco_ref',help='Set the reference that busco will use for analysis',default='metazoa')
parser.add_argument('--transrate_ref',help='A reference that transrate will use to evaluate the quality of your assembly.',default='')
parser.add_argument('-no_log',help='Pipeline will delete log files.',action='store_true')
parser.add_argument('-force',help='Use this flag to perform a fresh run of the pipeline. All steps will be executed regradless of what has already been performed.',action='store_true')
parser.add_argument('--cpu', help='Sets the process cap for execution. Default is 12. Use 0 to indicate no process cap should be used.',default=12,type=int)
parser.add_argument('--email',help='Pipeline will send emails informing you of runstate.')
parser.add_argument('--out_dir', help='Path to the ouput location. Defaults to assemblies directory inside pipeline',default=tf.PATH_ASSEMBLIES)
parser.add_argument('-o','--out_name', help='The name of the output directory to be made in out_dir. If unused, name will be inherited from input file names')
return parser.parse_args()
def annotation_args():
parser = argparse.ArgumentParser(description='Selected_tool : Annotation. Executing this tool will run asssembly quality assesment along with a series of tools designing to provide information about the assembled transcripts.')
parser.add_argument('tool_selector',help=argparse.SUPPRESS)
parser.add_argument('-test',help='Use this flag to test the pipeline.',action='store_true')
parser.add_argument('-a','--assembly',help='A fasta transcriptome assembly that will be used for computing expression levels.')
parser.add_argument('-blastplus',action='store_true',help='Use the blast+ tool suite instead of diamond to align your transcripts to the references.')
parser.add_argument('-uniref90',help='Use this flag to enable the uniref-90 blast runs as part of the annotation pipeline.',action='store_true')
parser.add_argument('-nr',help='Use this flag to enable the NR (non-redundant protein database) blast runs as part of the annotation pipeline. This could take a very long time to complete.',action='store_true')
parser.add_argument('-signalp',action='store_true',help='Use this flag to execute signalP during annotation. Only use if you have installed signalP.')
parser.add_argument('-tmhmm',action='store_true',help='Use this flag to execute tmhmm during annotation. Only use if you have installed tmhmm.')
parser.add_argument('-rnammer',action='store_true',help='Use this flag to execute rnammer during annotation. Only use if you have installed rnammer.')
parser.add_argument('-no_log',help='Pipeline will delete log files.',action='store_true')
parser.add_argument('-force',help='Use this flag to perform a fresh run of the pipeline. All steps will be executed regradless of what has already been performed.',action='store_true')
parser.add_argument('--cpu', help='Sets the thread cap for execution. Default is 12. Use 0 to indicate no process cap should be used.',default=12,type=int)
parser.add_argument('--email',help='Pipeline will send emails informing you of runstate.')
parser.add_argument('--out_dir', help='Path to the ouput location. Defaults to assemblies directory inside pipeline',default=tf.PATH_ASSEMBLIES)
parser.add_argument('-o','--out_name', help='The name of the output directory to be made in out_dir. If unused, name will be inherited from input file names')
return parser.parse_args()
def expression_args():
parser = argparse.ArgumentParser(description='Selected_tool : Expression. Executing this tool will run a series of differential expression analyses and sumarize the output.')
parser.add_argument('tool_selector',help=argparse.SUPPRESS)
parser.add_argument('-test',help='Use this flag to test the pipeline.',action='store_true')
parser.add_argument('-a','--assembly',help='A fasta transcriptome assembly that needs to be annotated.')
parser.add_argument('--csv', help='A CSV file specifying fastq input, basenames for DE, and factors for DE. See online documentation for details on formating the CSV file.',default=None)
parser.add_argument('--model', help='An optional list of comma seperated values used to run differential expression. This is particularly useful for refining Differential Expression runs as it allow you to use the same input CSV file and perform new comparisons.')
parser.add_argument('-no_log',help='Pipeline will delete log files.',action='store_true')
parser.add_argument('-force',help='Use this flag to perform a fresh run of the pipeline. All steps will be executed regradless of what has already been performed.',action='store_true')
parser.add_argument('--cpu', help='Sets the thread cap for execution. Default is 12. Use 0 to indicate no process cap should be used.',default=12,type=int)
parser.add_argument('--email',help='Pipeline will send emails informing you of runstate.')
parser.add_argument('--out_dir', help='Path to the ouput location. Defaults to assemblies directory inside pipeline',default=tf.PATH_ASSEMBLIES)
parser.add_argument('-o','--out_name', help='The name of the output directory to be made in out_dir. If unused, name will be inherited from input file names')
return parser.parse_args()
def manage_database_args():
parser = argparse.ArgumentParser(description='Selected_tool : Database Manager. This tool can be used to initialize MMT and update the databases.')
parser.add_argument('tool_selector',help=argparse.SUPPRESS)
parser.add_argument('-nr', action='store_true')
parser.add_argument('-uniref90', action='store_true')
parser.add_argument('-update', action='store_true')
# busco_argument
return parser.parse_args()
#####################____Argument_Testers____#####################
def global_setup(args):
if(not os.path.isdir(args.out_dir)):
try:
os.makedirs(args.out_dir)
except:
raise Exception('\n\nERROR : invalid out_dir argument. Path does not exist.')
if(args.cpu<=0):
args.cpu = float('inf')
tf.PATH_ASSEMBLIES = args.out_dir
if(args.tool_selector=='full'):
base = os.path.basename(args.csv)
#if args.assembly:
# base = os.path.basename(args.assembly)
base = base if('.' not in base) else '.'.join(base.split('.')[:-1])
if(args.tool_selector=='assembly'):
if(args.unpaired!=[]):
base = os.path.basename(args.unpaired[0])
base = base if('.' not in base) else '.'.join(base.split('.')[:-1])
else:
base = os.path.basename(args.fastq1[0])
base = base if('.' not in base) else '.'.join(base.split('.')[:-1])
base = base if('1' not in base) else '1'.join(base.split('1')[:-1])
if(args.tool_selector=='annotation' or args.tool_selector=='quality' or args.tool_selector=='expression'):
base = os.path.basename(args.assembly)
base = base if('.' not in base) else '.'.join(base.split('.')[:-1])
tf.NAME_OUT_DIR = args.out_name if(args.out_name!=None) else base
tf.NAME_ASSEMBLY = tf.NAME_OUT_DIR
tf.build_dir_task([]).run()
def check_full_args(args):
if(args.test):
args.out_name = args.out_name if(args.out_name!=None) else 'test_v2'
args.csv = tf.PATH_SCRIPTS+'/test_data/sample_info2.csv'
if(args.csv==None):
raise Exception('\n\nERROR : csv input is required for full execution of pipeline.')
if(not os.path.isfile(args.csv)):
raise Exception('\n\nERROR : Invalid csv argument. '+args.csv+' does not exist.')
args.subsample_size = args.subsample_size*10**6
handle_csv(args)
def check_assembly_args(args):
if(args.test):
args.out_name = args.out_name if(args.out_name!=None) else 'test_v2'
args.csv = tf.PATH_SCRIPTS+'/test_data/sample_info2.csv'
args.subsample_size = 10**15 if(args.subsample_size<=0) else args.subsample_size*10**6
if(args.csv):
handle_csv(args)
else:
args.unpaired = [] if(args.unpaired==None) else args.unpaired.split(',')
args.fastq1 = [] if(args.fastq1==None) else args.fastq1.split(',')
args.fastq2 = [] if(args.fastq2==None) else args.fastq2.split(',')
if(args.unpaired==[] and args.fastq1==[] and args.fastq2==[] and args.csv==[]):
raise Exception('\n\nERROR : No input files specified. Please specify input using either csv, or --fastq1 and --fastq2 or --unpaired.')
for f in chain(args.fastq1,args.fastq2,args.unpaired):
if(not os.path.isfile(f)):
raise Exception('\n\nERROR : Unable to find file : '+f)
for f1,f2 in zip(args.fastq1,args.fastq2):
if(not fastq_pair_check(f1,f2)):
raise Exception('\n\nERROR : '+f1+' cant be paired with '+f2+'.')
def check_quality_args(args):
if(args.test):
args.out_name = args.out_name if(args.out_name!=None) else 'test_v2'
args.assembly = tf.PATH_SCRIPTS+'/test_data/test_v2.fasta'
args.unpaired = [] if(args.unpaired==None) else args.unpaired.split(',')
args.fastq1 = [] if(args.fastq1==None) else args.fastq1.split(',')
args.fastq2 = [] if(args.fastq2==None) else args.fastq2.split(',')
if(args.assembly==None):
raise Exception('\n\nERROR : No input assembly specified. Please specify an input assembly using --assembly.')
if(not os.path.isfile(args.assembly)):
raise Exception('\n\nERROR : Invalid assembly argument. '+args.assembly+' does not exist.')
for f in chain(args.fastq1,args.fastq2,args.unpaired):
if(not os.path.isfile(f)):
raise Exception('\n\nERROR : Unable to find file : '+f)
for f1,f2 in zip(args.fastq1,args.fastq2):
if(not fastq_pair_check(f1,f2)):
raise Exception('\n\nERROR : '+f1+' cant be paired with '+f2+'.')
def check_annotation_args(args):
if(args.test):
args.out_name = args.out_name if(args.out_name!=None) else 'test_v2'
args.assembly = tf.PATH_SCRIPTS+'/test_data/test_v2.fasta'
if(args.assembly==None):
raise Exception('\n\nERROR : No input assembly specified. Please specify an input assembly using --assembly.')
if(not os.path.isfile(args.assembly)):
raise Exception('\n\nERROR : Invalid assembly argument. '+args.assembly+' does not exist.')
def check_expression_args(args):
if(args.test):
args.out_name = args.out_name if(args.out_name!=None) else 'test_v2'
args.csv = tf.PATH_SCRIPTS+'/test_data/sample_info2.csv'
args.assembly = tf.PATH_SCRIPTS+'/test_data/test_v2.fasta'
if(args.assembly==None):
raise Exception('\n\nERROR : No input assembly specified. Please specify an input assembly using --assembly.')
if(not os.path.isfile(args.assembly)):
raise Exception('\n\nERROR : Invalid assembly argument. '+args.assembly+' does not exist.')
if(args.csv==None):
raise Exception('\n\nERROR : csv input is required for full execution of pipeline.')
if(not os.path.isfile(args.csv)):
raise Exception('\n\nERROR : Invalid csv argument. '+args.csv+' does not exist.')
handle_csv(args)
def fastq_pair_check(fastq1,fastq2):
'''
count1 = 0
with open(fastq1) as f:
for line in f:
count1+=1
count2 = 0
with open(fastq2) as f:
for line in f:
count2+=1
return count1==count2
'''
return True
def handle_csv(args):
f = open(args.csv)
head = f.readline().rstrip().split(',')
args.paired_names=[]
args.unpaired_names=[]
args.fastq1=[]
args.fastq2=[]
args.unpaired=[]
if(args.tool_selector == 'expression' or args.tool_selector=='full'):
if(args.model==None):
args.model = ' '.join(head[3:])
for line in f:
fields = line.rstrip().split(',')
if(fields[2]!=''):
args.paired_names.append(fields[0])
args.fastq1.append(os.path.abspath(fields[1]))
args.fastq2.append(os.path.abspath(fields[2]))
else:
args.unpaired_names.append(fields[0])
args.unpaired.append(os.path.abspath(fields[1]))
f.close()
for f in chain(args.fastq1,args.fastq2,args.unpaired):
if(not os.path.isfile(f)):
raise Exception('\n\nERROR : Unable to find file : '+f)
for f1,f2 in zip(args.fastq1,args.fastq2):
if(not fastq_pair_check(f1,f2)):
raise Exception('\n\nERROR : '+f1+' cant be paired with '+f2+'.')
def gen_sample_info(args):
args.sample_info = tf.GEN_PATH_EXPRESSION_FILES()+'/sample_info.tsv'
si = open(args.sample_info,'w')
f = open(args.csv)
for line in f:
temp = line.split(',')
si.write('\t'.join([temp[0]]+temp[3:]))
si.close()
f.close()
######################___go_functions___######################
def go_assembly(args, dep):
return gen_assembly_supervisor(
args.fastq1, args.fastq2, args.unpaired, dep,args.no_trim,
args.rnaspades, args.no_rmdup, args.subsample_size,
args.cpu, args.subsample_seed, args.trinity_normalization,
args.truncate, args.trimmomatic)
def go_quality(args, dep):
return gen_quality_supervisor(
args.fastq1, args.fastq2, args.unpaired, dep, args.busco_ref,
args.cpu, args.cegma, args.transrate_ref)
def go_annotation(args, dep):
return gen_annotation_supervisor(
args.cpu, args.uniref90, args.nr, args.blastplus, args.signalp,
args.tmhmm, args.rnammer, dep)
def go_expression(args, dep):
return gen_expression_supervisor(
args.fastq1, args.fastq2, args.paired_names, args.unpaired,
args.unpaired_names, args.cpu, args.sample_info, args.model, dep)
#####################____Main_Modules____#####################
def run_full(args):
check_full_args(args)
global_setup(args)
gen_sample_info(args)
supers = []
deps = []
assembly_super = go_assembly(args, [])
supers.append(assembly_super)
quality_super = go_quality(args, [assembly_super])
supers.append(quality_super)
annotation_super = go_annotation(args, [assembly_super])
supers.append(annotation_super)
expression_super = go_expression(args, [assembly_super])
supers.append(expression_super)
run_supers(args, supers)
#try to allow assembly arg...
#if args.assembly:
# cp = tf.cp_assembly_task(args.assembly, [])
# supers.append(cp)
# deps = [cp]
# else:
# assembly_super = go_assembly(args, [])
# supers.append(assembly_super)
# deps = [assembly_super]
# quality_super = go_quality(args, deps)
# supers.append(quality_super)
# annotation_super = go_annotation(args, deps)
# supers.append(annotation_super)
# expression_super = go_expression(args, deps)
# supers.append(expression_super)
# run_supers(args, supers)
def run_assembly(args):
check_assembly_args(args)
global_setup(args)
supers = []
assembly_super = go_assembly(args, [])
supers.append(assembly_super)
#quality_super = go_quality(args, [assembly_super]) # now we always run quality with assembly
#supers.append(quality_super)
run_supers(args, supers)
def run_quality(args):
check_quality_args(args)
global_setup(args)
supers = []
cp = tf.cp_assembly_task(args.assembly, [])
supers.append(cp)
quality_super = go_quality(args, [cp])
supers.append(quality_super)
run_supers(args, supers)
def run_annotation(args):
check_annotation_args(args)
global_setup(args)
supers = []
cp = tf.cp_assembly_task(args.assembly, [])
supers.append(cp)
annotation_super = go_annotation(args, [cp])
supers.append(annotation_super)
run_supers(args, supers)
def run_expression(args):
check_expression_args(args)
global_setup(args)
gen_sample_info(args)
supers = []
cp = tf.cp_assembly_task(args.assembly, [])
supers.append(cp)
expression_super = go_expression(args, [cp])
supers.append(expression_super)
run_supers(args, supers)
def run_databases(args):
busco_opts = {'arthropoda': args.busco_arthopoda, 'metazoa': args.busco_metazoa, 'vertebrata': False,
'eukaryota': False, 'fungi': False, 'bacteria': False,
'plants': False}
db_install(args.nr, args.uniref90, not args.update, busco_opts)
def run_supers(args, supers):
run_log = os.path.join(tf.GEN_PATH_DIR(), 'run.log')
total = Supervisor(tasks=supers, cpu=args.cpu, force_run=args.force, log=run_log, email=args.email)
try:
total.run()
except:
build_log(args, total.task_status)
raise
build_log(args, total.task_status)
def build_log(args, task_status):
master_log = os.path.join(tf.GEN_PATH_DIR(), 'master.log')
f = open(master_log, 'a')
f.write('##################__'+args.tool_selector+'__##################\n')
skipped_tasks = {t: task_status[t] for t in task_status if(task_status[t]['state'] == Supervisor.STATE_SKIPPED)}
completed_tasks = {t: task_status[t] for t in task_status if(task_status[t]['state'] == Supervisor.STATE_FINISHED)}
failed_tasks = {t: task_status[t] for t in task_status if(task_status[t]['state'] == Supervisor.STATE_ERR)}
removed_tasks = {t: task_status[t] for t in task_status if(task_status[t]['state'] == Supervisor.STATE_REMOVED)}
f.write('\nThe following jobs were detected as already having been '
'completed. They were not executed as part of this pipeline\'s '
'execution. Output from previous runs was used instead\n')
for t in skipped_tasks:
f.write('\t'+t.name+'\n')
f.write('\nThe following jobs were exectued succesfully.\n')
for t in completed_tasks:
f.write('\t'+t.name+' : '+completed_tasks[t]['message']+'\n')
f.write('\nThe following jobs encountered an unexpected error during execution.\n')
for t in failed_tasks:
f.write('\t'+t.name+' : '+failed_tasks[t]['message']+'\n')
f.write('\nDue to the above errors, the following jobs could not be started.\n')
for t in removed_tasks:
f.write('\t'+t.name+'\n')
f.write('\n\n')
f.close()
if(__name__ == '__main__'):
err_str = ('\n\nError : Unable to identify what tool should be executed. '
'Valid arguments are "full", "assembly", "annotation", "quality" '
'or "expression".')
args = master_args()
tool = args.tool_selector.lower()
if(tool != 'a'):
if(tool == 'full'[:len(tool)]):
args = full_args()
args.tool_selector = 'full'
run_full(args)
if(tool == 'assembly'[:len(tool)]):
args = assembly_args()
args.tool_selector = 'assembly'
run_assembly(args)
if(tool == 'annotation'[:len(tool)]):
args = annotation_args()
args.tool_selector = 'annotation'
run_annotation(args)
if(tool == 'expression'[:len(tool)]):
args = expression_args()
args.tool_selector = 'expression'
run_expression(args)
if(tool == 'quality'[:len(tool)]):
args = quality_args()
args.tool_selector = 'quality'
run_quality(args)
if(tool == 'databases'[:len(tool)]):
args = manage_database_args()
args.tool_selector = 'databases'
run_databases(args)
#raise Exception(err_str)
else:
raise Exception(err_str)