-
Notifications
You must be signed in to change notification settings - Fork 8
Expand file tree
/
Copy pathutils.py
More file actions
271 lines (215 loc) · 7.87 KB
/
Copy pathutils.py
File metadata and controls
271 lines (215 loc) · 7.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
"""
Purpose of the script: configures logging
"""
import argparse
import logging
import os
import random
import shutil
import string
import sys
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Tuple
import pandas as pd
import toml
import src.params as params
def configure_pipeline(
output_dir: str, hash_length: int, custom_hash: Optional[str] = None
) -> Tuple[logging.Logger, List[str]]:
"""Configure the pipeline run
Parameters
----------
output_dir : str
Location to store outputs (including logs)
hash_length : int
Length of hash to use in pipeline folder
custom_hash : Optional[str], optional
Specify hash to use, by default None
Returns
-------
Tuple[logging.Logger, List[str]]
Logger object and output folders
Raises
------
ValueError
If empty custom hash supplied
"""
if custom_hash is not None:
if len(custom_hash) == 0:
raise ValueError("Custom hash must be of length > 0")
if len(custom_hash) > hash_length:
custom_hash = custom_hash[:hash_length]
pipeline_hash = get_unique_folder_name(hash_length, custom_hash)
logger = configure_logging(output_dir, pipeline_hash)
output_folders = create_output_folder(output_dir, pipeline_hash)
return (logger, output_folders)
def configure_logging(output_dir: str, hash_id: str) -> logging.Logger:
"""Set up logging format and location to store logs
Store logs in a secure location (e.g. IC Green)
Do not store logs on your local machine as they may contain traces of data.
Args:
output_dir: directory to store logs
hash_id: pipeline hash
Returns:
(logging.Logger) logger to add detail to
"""
log_folder = Path(output_dir).joinpath("logs")
if not os.path.exists(Path(log_folder)):
os.makedirs(Path(log_folder))
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(levelname)s -- %(filename)s:\
%(funcName)5s():%(lineno)s -- %(message)s",
handlers=[
logging.FileHandler(log_folder / f"{hash_id}.log"),
logging.StreamHandler(sys.stdout),
], # Add second handler to print log message to screen
)
logger = logging.getLogger(__name__)
logger.info("Configured logging.")
logger.info(f"Run under hash: {hash_id}.")
logger.info(f"Starting run at:\t{datetime.now().time()}")
logger.info(f"Starting run at:\t{datetime.now().time()}")
return logger
def get_config(toml_path="config.toml") -> dict:
"""Gets the config toml from the root directory and returns it as a dict.
Returns:
Dict: A dictionary containing details of the database, paths, etc.
Example:
from shmi_improvement.utilities.helpers import get_config
config = get_config()
"""
return toml.load(toml_path)
def random_string(length: int) -> str:
"""Generates a random string
Args:
length (int): The length of the string
Returns:
(str) A random string
"""
pool = string.hexdigits
return "".join(random.choice(pool) for i in range(length))
def get_unique_folder_name(length: int, custom_hash: Optional[str] = None) -> str:
"""Generates a unique folder name
Args:
length (int): Length of random hash
custom_hash (Optional[str]): Custom hash to use
Returns:
str: A string containing a date and random, or custom hash.
"""
now = datetime.now()
now_str = now.strftime("%d-%m-%Y_%H-%M-%S")
if custom_hash is not None:
print("Using custom hash")
folder_string = f"{now_str}_{custom_hash}"
else:
folder_string = f"{now_str}_{random_string(length)}"
return folder_string
def create_output_folder(output_path: str, output_folder: str):
"""Creates a new folder in the specified output path.
Args:
output_path (str):
Location of output
output_folder (str):
Name of folder name to store pipeline run output
Returns:
list[str]: an array of paths
"""
if not os.path.exists(Path(output_path)):
os.makedirs(Path(output_path))
project_output_path = Path(output_path).joinpath(output_folder)
project_output_path_outputs = Path(project_output_path).joinpath(
params.output_folder
)
project_output_path_reports = Path(project_output_path).joinpath(
params.report_folder
)
project_output_path_transformations = Path(project_output_path).joinpath(
params.transformations_folder
)
os.mkdir(project_output_path)
os.mkdir(project_output_path_outputs)
os.mkdir(project_output_path_reports)
os.mkdir(project_output_path_transformations)
print(f"Folders created in:\n{project_output_path}")
shutil.copy2(
src=(Path.cwd().joinpath("config.toml")),
dst=project_output_path.joinpath("config.txt"),
)
print("Config copied and saved.")
return [
project_output_path,
project_output_path_outputs,
project_output_path_reports,
project_output_path_transformations,
]
def copy_outputs(old_file_path: str, new_file_path: str, file_name: str) -> None:
shutil.copy2(
src=Path(old_file_path).joinpath(file_name),
dst=Path(new_file_path).joinpath(file_name),
)
def check_file_for_duplicates(df: pd.DataFrame, file_name: str, col: str) -> None:
"""Checks a field of a dataframe for duplicate values.
The dataframe should contain data linked to a file.
Args:
df (pd.DataFrame):
data from file
file_name (str):
name of file so that the user knows which file to check
col (str):
the field to check for duplicates
Raises:
ValueError if any col values appear more than once in the dataframe
Returns:
None
"""
if df.duplicated(subset=[col]).any():
raise ValueError(
f"""Your dataset: {file_name} contains duplicate {col} values. \n
This pipeline requires distinct {col} values."""
)
return None
def check_121_mappings(
df: pd.DataFrame, file_name: str, left_field: str, right_field: str
) -> int:
"""Checks whether a field is mapped to more than one value in another field.
The dataframe should contain data linked to a file.
Args:
df (pd.DataFrame):
data from file
file_name (str):
name of file so that the user knows which file to check
left_field (str):
the field to map
right_field (str):
the field which may contain multiple values
Raises:
ValueError if left_field is mapped to more than one value of right_field
Returns:
None
"""
df_count_mappings = df.groupby(left_field)[right_field].nunique()
max_mapping = int(df_count_mappings.max())
if max_mapping > 1:
raise ValueError(
f"""Your dataset: {file_name} contains incorrect mappings. \n
It contains multiple {right_field} values for each {left_field}. \n
This pipeline requires 1 {right_field} for each {left_field}"""
)
return None
def parse_cli_args() -> argparse.Namespace:
"""Parses arguments supplied at the command line
Returns
-------
argparse.Namespace
argparse object with hash attribute
"""
parser = argparse.ArgumentParser(
description="Pipeline to calculate Local Authority Peers"
)
parser.add_argument(
"--hash", default=None, help="Supply a custom hash to store your pipeline."
)
args = parser.parse_args()
return args