-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathprocessing.py
More file actions
437 lines (334 loc) · 14.5 KB
/
Copy pathprocessing.py
File metadata and controls
437 lines (334 loc) · 14.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
import os
import re
import pickle
import datetime
import subprocess
import numpy as np
import pandas as pd
from tqdm.autonotebook import tqdm
from copy import deepcopy
from pathlib import Path
from itertools import chain
from collections import defaultdict
NONALNUM_PATTERN = re.compile('[\W_]+')
def strip_chars(string):
return NONALNUM_PATTERN.sub('', string)
def head(file_name, n_lines=20, print_out=True, line_nums=False):
"""Get the first n_lines lines of a file. Print if print_out=True, else return as list. Works on UNIX systems"""
assert type(n_lines) == int
n_lines = str(n_lines)
if os.path.splitext(file_name)[-1] in ['.gz', '.zip']:
zcat_p = subprocess.Popen(['zcat', file_name], stdout=subprocess.PIPE)
head_p = subprocess.Popen(['head', '-n', n_lines], stdin=zcat_p.stdout, stdout=subprocess.PIPE)
zcat_p.stdout.close()
output = head_p.communicate()[0].decode('utf-8')
else:
output = subprocess.Popen(['head', '-n', n_lines, file_name], stdout=subprocess.PIPE).stdout.read().decode('utf-8')
if not print_out:
return output
if line_nums:
pad = int(np.log10(int(n_lines)))+1
fmt_str = '{:' + '{}'.format(pad) + '.0f}:'
split_out = output.rstrip('\n').split('\n')
for i, line in enumerate(split_out):
print(fmt_str.format(i), line)
else:
print(output)
def cammel_to_underscore(name):
"""Converts CammelCaseNames to underscore_separated_names"""
# Get the indicies of capital letters
idx_to_change = []
for i, letter in enumerate(name):
if letter != letter.lower():
idx_to_change.append(i)
# Ensure first section is grabbed when starting with lower case e.g. cammelCaseExamples
idx_to_change = [0] + idx_to_change if 0 not in idx_to_change else idx_to_change
# If strings of caplital letters in a row (e.g. OrchidIDs)
# Start with only the first of the string and remove the rest...
prev_idx = 0
to_remove = []
for idx in idx_to_change:
if idx - prev_idx == 1:
to_remove.append(idx)
prev_idx = idx
idx_to_change = [x for x in idx_to_change if x not in to_remove]
# Build the new name
out_name = ''
for i, start_idx in enumerate(idx_to_change):
if i+1 < len(idx_to_change):
end_idx = idx_to_change[i+1]
out_name += name[start_idx:end_idx].lower() + '_'
else:
out_name += name[start_idx:].lower()
if not out_name:
return name
return out_name
def remove_lead_chars(name):
"""Strips whitespace and special characters from the start of a string"""
start = 0
while not name[start].isalpha():
start+=1
return name[start:]
def remove_end_chars(name):
"""Strips whitespace and special characters from the end of a string"""
end = len(name)
while not name[end-1].isalnum():
end-=1
return name[:end]
def strip_special_chars(name):
"""Removes most special characters from a string, keeps [' ', '-', or '_']"""
return ''.join([c for c in name if c.isalnum() or c in [' ', '-', '_']])
def regularize_colname(name):
"""Regularize a string to a Pandas queryable name, with underscore separation"""
out_name = remove_lead_chars(name)
out_name = remove_end_chars(out_name)
out_name = strip_special_chars(out_name)
if out_name.isidentifier():
out_name = cammel_to_underscore(out_name)
out_name = out_name.replace(' ', '_').replace('-', '_')
return out_name.lower()
def regularize_colnames(col_names):
"""Regularize a list of column names. See regularize_colname"""
return [regularize_colname(c) for c in col_names]
def split_col(col, char):
"""Splits a Series by a characters, into a new series of equal length with each element as a list"""
return col.apply(lambda s: s.split(char) if type(s) == str else [s])
def expand_split_col(col_split):
"""
Expands a Series that contains a list into a new DataFrame with single item per row, and the original index
contained in a new column 'old_idx'
Example
1 [5, 4, 3]
3 [2, 5]
5 [1]
name='numbers'
would become
old_idx, numbers
1, 5
1, 4
1, 3
3, 2
3, 5
5, 1
"""
col_name = col_split.name
col_split = col_split.to_frame().reset_index()
old_idx = []
new_col = []
for row in col_split.itertuples():
for item in row[2]:
old_idx.append(row[1])
new_col.append(item)
return pd.DataFrame({'old_idx': old_idx, col_name: new_col})
def expand_df_on_col(df, col_name):
df_out = df.copy()
# ensure all elements are lists, not just a subset.
df_out[col_name] = df_out[col_name].apply(lambda e: [e] if type(e) != list else e)
exp_col = expand_split_col(df_out[col_name])
col_order = df_out.columns
df_out = df_out.reset_index()
df_out = df_out.rename(columns={'index': 'old_idx'})
df_out = pd.merge(df_out.drop(col_name, axis=1), exp_col, on='old_idx', how='outer')
return df_out.drop('old_idx', axis=1)[col_order]
def expand_col_on_char(df, col_name, char):
"""
Expands rows in a dataframe due to a column where elements contain multiple values separated by a character,
resulting in only one element per row in the new column
For example, a column of pipe separated Pubmed IDs (in .csv format):
letter, pmid
a, 12805067|17698565|18279049|21329777
b, 10072544|12721113
Would become:
letter, pmid
a, 12805067
a, 17698565
a, 18279049
a, 21329777
b, 10072544
b, 12721113
"""
# Copy df and get column order
df_out = df.copy()
col_order = df_out.columns
# Split the desired column on the desired character and expand the rows
col_split = split_col(df_out[col_name], char)
col_split = expand_split_col(col_split)
# Make the old index avaliable for merging
df_out = df_out.reset_index()
df_out = df_out.rename(columns={'index': 'old_idx'})
df_out = pd.merge(df_out.drop(col_name, axis=1), col_split, on='old_idx', how='outer')
return df_out.drop('old_idx', axis=1)[col_order]
def map_to_parent(df, parent_map, col_name):
"""Map a column to its parent value"""
df_out = df.copy()
out_name = 'parent_'+col_name
df_out[out_name] = df_out[col_name].map(parent_map)
return df_out
def make_parent_map(df, child_id_col, parent_id_col, name_col):
"""
Takes dataframe with identifiers for parent and child, and name for child and produces a map dict from
child name to parent name
"""
map_df = pd.merge(df, df[[name_col, child_id_col]], how='left', left_on=parent_id_col, right_on=child_id_col,
suffixes=('', '_parent'))
par_name_col = name_col+'_parent'
# Fills in root nodes with its own name
map_df[par_name_col] = map_df[par_name_col].fillna(map_df[name_col])
return map_df.set_index(name_col)[par_name_col].to_dict()
def prepend_direction_to_map(mapper, directions=iter(()), char=''):
"""
In a dict, will prepend all elements of `directions` to all keys and values, sparated by `char`
e.g. a dict of {'metabolic processing': 'metabolic processing',
'acetylation': 'metabolic processing'}
And directions ['increasing', 'decreasing'] with char '^'
becomes:
{'increases^metabolic processing': 'increases^metabolic processing',
'increases^acetylation': 'increases^metabolic processing',
'decreases^metabolic processing': 'decreases^metabolic processing',
'decreases^acetylation': 'decreases^metabolic processing'}
"""
out_map = dict()
for k, v in mapper.items():
for d in directions:
new_k = d+char+k
new_v = d+char+v
out_map[new_k] = new_v
return out_map
def get_parent_to_child_map(df, parent_col, child_col, sep='|', verbose=False):
"""
Gets a map from the parent ID to IDs of all children (and children of children etc.)
Source dataframe (EG, CTD)
"""
# Initialize the map
par_to_children = defaultdict(set)
expanded = expand_col_on_char(df.dropna(subset=[parent_col]), parent_col, sep)
# Get mappings from child to local parents
for row in expanded.itertuples():
par_to_children[getattr(row, parent_col)].add(getattr(row, child_col))
# Extend the map... parents map to local children, but need mapping to the children of children, too
par_to_children = extend_map(par_to_children, verbose)
# Discard empty values (default_dict will add an empty one when simply accessed)
par_to_children = {k: v for k, v in par_to_children.items() if len(v) > 0}
return par_to_children
def extend_map(mapper, verbose=False):
"""
Takes a dict mapper of a tree (e.g. parents to children) a extends down so parents map to all
intermediate children, terminating at a leaf.
"""
prev_total_mappings = 0
total_mappings = len(list(chain(*mapper.values())))
num_iter = 0
i = 0
while (total_mappings != prev_total_mappings):
prev_total_mappings = len(list(chain(*mapper.values())))
current_map = deepcopy(mapper)
for k, v in current_map.items():
for val in v:
mapper[k].update(mapper[val])
total_mappings = len(list(chain(*mapper.values())))
if verbose:
num_iter += total_mappings
# Print an update every 10 Million iterations (approx)
if i == 0 or np.floor(num_iter / 1e7) - np.floor((num_iter - total_mappings) / 1e7) > 0:
print('Iter: {}, Total Mappings: {:,}'.format(i, total_mappings))
i += 1
return mapper
def char_combine_col(col, char='|'):
"""
Converts a column to a cell, splitting elements within that column along a character, dedupcating all elements,
then joining across that character on that charcter.
e.g. 1 "123456|102934"
2 "123456"
3 "102934|432452"
4 "201945"
becomes:
"123456|102934|432452|201945"
"""
elems = []
for elem in col:
elems += str(elem).split(char)
return char.join(list(set([e for e in elems if e != 'nan'])))
def char_combine_dataframe_rows(df, char='|'):
return df.apply(lambda col: char_combine_col(col, char))
def combine_group_rows_on_char(df, group_on, combine_cols=None, char='|'):
"""
Performs a Groupby on a dataframe and then converts each group into a single row, joinned by a character `char`
Primarly suppports grouping on columns, other methods have not been tested.
:param df: The dataframe to group
:param group_on: the column name or list of column names to group by
:param combine_cols: a list of column names to combine with a character, if None, will combine all columns.
can save computation time to provide only the columns of interest for combination
:param char: the character to combine the columns with. Defaults to a `|` character.
:return: Dataframe with 1 row per group, and information of different rows joined by given character.
"""
col_order = df.columns
if type(group_on) in (str, int, float):
group_on = [group_on]
grouped = df.groupby(group_on)
if combine_cols is None:
combine_cols = find_cols_with_multi_values(grouped)
out_df = grouped.first()
for col in tqdm(combine_cols, desc='total_progress'):
tqdm.pandas(desc=col)
out_df[col] = grouped[col].progress_apply(char_combine_col, char=char)
return out_df.reset_index()[col_order]
def find_cols_with_multi_values(grouped):
"""
In a Pandas Groupby object, determines which columns have more than one value per group
"""
multi_val_cols = (grouped.nunique() > 1).sum()
multi_val_cols = multi_val_cols[multi_val_cols > 0].index.tolist()
return multi_val_cols
def make_child_to_root_map(df, parent_col, child_col, sep='|', verbose=False):
"""
In a dataframe where one column is child identfiers, and another parent,
creates a map from child to root.
"""
root_concepts = df[df[parent_col].isnull()][child_col].tolist()
parent_to_child = get_parent_to_child_map(df, parent_col, child_col, sep, verbose)
child_to_parent = dict()
for k, v in parent_to_child.items():
for val in v:
if k in root_concepts:
child_to_parent[val] = k
for r in root_concepts:
child_to_parent[r] = r
return child_to_parent
def convert_abbrev_mapper_to_full(mapper, map_df, abbrev, full):
"""
With a mapping file from abbrev to abbrev, and a datafame that has abbrev to full mappings, produces
a mapper from full to full.
"""
abbrev_to_name = map_df.set_index(abbrev)[full].to_dict()
return {abbrev_to_name[k]: abbrev_to_name[v] for k, v in mapper.items()}
def load_api_results(res_file_name, re_scrape=False, scrape_function=lambda **f: None, **kwargs):
"""
Loads results from an api query. If file does not exit, or rescrape is true, and an api function is passed,
The API will rescrape the data.
:param res_file_name: string or Path, the filename to load or save to (must be .pkl for now).
If `{}` is included in the filename, will glob on that point in the filename and take the highest value
(newest if that happpens to correspond to a date). Also, if a download is performed and the file is saved,
a `{}` will be filled with the current date in the output file name.
:param rescrape: bool, re-download the data even if a file matching the input name already exists.
:param scrape_function: the function to be called if the datafile does not exist, or if re_scrape is true.
:param **kwargs: any keyword arguments for the scrape funtion.
:return: Data either loaded from disk or scraped from an api.
"""
# Make sure filname is string
if not isinstance(res_file_name, Path):
res_file_path = Path(res_file_name).resolve()
else:
res_file_path = res_file_name
res_file_name = str(res_file_path)
dump_files = list(res_file_path.parent.glob(res_file_path.name.format('*')))
if len(dump_files) < 1 or re_scrape:
# Scrape (or re-scrape) the API and save
res = scrape_function(**kwargs)
if res:
with open(res_file_name.format(datetime.datetime.now().strftime("%Y-%m-%d")), 'wb') as f_out:
pickle.dump(res, f_out)
else:
# Load the most recent previously saved dump
dump_file = sorted(dump_files, reverse=True)[0]
res = pickle.load(open(dump_file, 'rb'))
return res