Repository navigation
Expand file tree
/
Copy pathrandom_subset.py
More file actions
191 lines (168 loc) · 6.43 KB
/
Copy pathrandom_subset.py
File metadata and controls
191 lines (168 loc) · 6.43 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
'''
filename : randomSubset.py
Author : Nolan Hartwick
Date : 4/7/15
Descritption : randomSubset.py is a sampling algorithm for paired fastq files.
It is safe to import and can also be used as a script.
Call Method : python randomSubset.py fastqfile1 fastqfile2 linecount numbins samplesize target1 target2
Use "python randomSubset.py -h" for help/usage description.
USAGE MESSAGE :
usage: randomSubset.py [-h]
SourceFile1 SourceFile2 Linecount Numbins SampleSize
TargetFile1 TargetFile2
Descritption: Fastq file sampling algorithm.
positional arguments:
SourceFile1 The fastq file that is paired with SourceFile2 and that we
desire a sample of.
SourceFile2 The fastq file that is paired with SourceFile1 and that we
desire a sample of.
Linecount The number of lines in either SourceFile1 or SourceFile2. Enter
0 if you want randomSubset.py to compute Linecount.
Numbins The number of bins to divide the Fastq file into, before taking
subsets of each.
SampleSize The desired samplesize in number of fastq entries to include in
sample, or in percent as a number between 0 and 1.
TargetFile1 An output filename. The sample from SourceFile1 writes to
TargetFile1.
TargetFile2 An output filename. The sample from SourceFile2 writes to
TargetFile2.
optional arguments:
-h, --help show this help message and exit
'''
import argparse
import random
from itertools import chain
import time
''' Generates a psuedorandom subset of paired fastq files filename1 and
filename2. It first seperates all entries into numbins sequential subsets.
A random sample is then taken from each of those subsets.
Params:
filename1- Path to a fastq file
filename2- Path to the fastq file that is paired with filename1
linecount- the linecount of filename1/2
numbins- the number of bins to divide the files into
samplesize- the size of the sample in number of fastq files
filename1target- path to write sample from filename1 to
filename2target- path to write sample from filename2 to
'''
def fq_parser(fq):
f = open(fq)
count = 0
ret = []
for line in f:
ret.append(line)
count+=1
if(count%4==0):
yield ret
ret = []
def paired_fq_parser(fq1,fq2):
iters = [zip(fq_parser(f1),fq_parser(f2)) for f1,f2 in zip(fq1,fq2)]
for e1,e2 in chain(*iters):
yield (e1,e2)
def GenRandomizedSubset_v2(fq1,fq2,numbins,samplesize,target1,target2,seed=None):
entry_count = 0
for e in paired_fq_parser(fq1,fq2):
entry_count+=1
if(samplesize<1 and samplesize>0):
samplesize = samplesize*entry_count
if(samplesize<=0):
samplesize = entry_count*2
p_val = float(samplesize)/entry_count
t1 = open(target1,'w')
t2 = open(target2,'w')
seed_time = seed if(seed!=None) else str(time.time())
print("Seed Used For RNG : "+str(seed_time))
random.seed(seed_time)
for e1,e2 in paired_fq_parser(fq1,fq2):
if(random.random()<p_val):
for line in e1:
t1.write(line)
for line in e2:
t2.write(line)
def GenRandomizedSubset(filename1,filename2,linecount,numbins,samplesize,filename1target,filename2target):
entries = linecount/4
binsize = entries/numbins
ranges = [[int(round(binsize*x)),int(round(binsize*(x+1)))] for x in range(numbins)]
if(ranges[-1][1]!=entries):
ranges[-1][1] = entries
#define number of samples needed from each bin
sampleinbins = int(samplesize/numbins)
dif = samplesize - sampleinbins*numbins
samplevals = [sampleinbins for x in range(numbins)]
safetoincrement = [x for x in range(numbins) if(ranges[x][1]-ranges[x][0]>sampleinbins)]
for x in random.sample(safetoincrement,dif):
samplevals[x]+=1
#choose the entries to include in final subset
ret = []
for x in range(len(ranges)):
t=random.sample(range(ranges[x][0],ranges[x][1]),min(samplevals[x],ranges[x][1]-ranges[x][0]) )
t.sort()
ret+=t
#generate subsets
dump(filename1,ret,filename1target)
dump(filename2,ret,filename2target)
''' Takes in a fastq file to read from, and a list of entries to pull and
write to target
'''
def dump(filename,entries,target):
f = open(filename)
g = open(target,'w')
count = 0
pos=0
flag=False
for line in f:
if(count%4==0):
if(pos>=len(entries)):
break
elif(count/4==entries[pos]):
pos+=1
flag=True
else:
flag=False
if(flag):
g.write(line)
count+=1
f.close()
g.close()
''' Checks whether filename is a valid place to read from.
'''
def rfcheck(filename):
argparse.FileType('r')(filename)
return filename
''' Checks whether filname is a valid place to write to.
'''
def wfcheck(filename):
t = argparse.FileType('w')(filename)
return filename
''' Computes the number of lines in filename.
'''
def lc(filename):
f = open(filename)
count = 0
for line in f:
count+=1
return count
f.close()
#Acts like a main method
if(__name__=='__main__'):
parser = argparse.ArgumentParser(description="Descritpion: Fastq file sampling script.")
parser.add_argument('-1','--fastq1', help='a list of comma seperated fastq files paired with the files in fastq2.')
parser.add_argument('-2','--fastq2', help='a list of comma seperated fastq files paired with the files in fastq1.')
parser.add_argument('-n','--numbins', type=int, help = 'The number of bins to divide the Fastq entries into, before taking subsets of each.')
parser.add_argument('-s','--sample_size', type=float, help='The desired samplesize in number of fastq entries to include in sample, or in percent as a number between 0 and 1.')
parser.add_argument('-t1','--target_file1', type=wfcheck, help='An output filename. The sample from SourceFile1 writes to TargetFile1.')
parser.add_argument('-t2','--target_file2', type=wfcheck, help='An output filename. The sample from SourceFile2 writes to TargetFile2.')
parser.add_argument('--seed',help='The seed value to be used by the random number generator.',default=str(time.time()))
args = parser.parse_args()
args.fastq1 = args.fastq1.split(',')
args.fastq2 = args.fastq2.split(',')
GenRandomizedSubset_v2(args.fastq1,args.fastq2,args.numbins,args.sample_size,args.target_file1,args.target_file2,args.seed)
'''
if(args.Linecount==0):
args.Linecount = lc(args.SourceFile1)
if(args.SampleSize<1):
args.SampleSize = int(round(args.Linecount/4*args.SampleSize))
else:
args.SampleSize=int(args.SampleSize)
GenRandomizedSubset(args.SourceFile1,args.SourceFile2,args.Linecount,args.Numbins,args.SampleSize,args.TargetFile1,args.TargetFile2)
'''