-
Notifications
You must be signed in to change notification settings - Fork 7
Expand file tree
/
Copy pathoutput_checker.py
More file actions
371 lines (334 loc) · 12.7 KB
/
Copy pathoutput_checker.py
File metadata and controls
371 lines (334 loc) · 12.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
import pandas as pd
import numpy as np
def check_null(df: pd.DataFrame, columns_to_be_checked: list) -> bool:
"""
Checks a pandas dataframe for null values
This function takes a pandas dataframe supplied as an argument and returns a integer value representing any null values found within the columns to check
Parameters
----------
df : pd.DataFrame
Dataframe to read
columns_to_be_checked: list
Given dataframe columns to be checked for null values
Returns
-------
out : int
The number of null values found in the given columns
Examples
--------
>>> check_null(df = pd.DataFrame({'col1': [1,2], 'col2': [3,4]}), columns_to_be_checked = ['col1', 'col2'])
0
>>> check_null(df = pd.DataFrame({'col1': [1,np.nan], 'col2': [3,4]}), columns_to_be_checked = ['col1'])
1
"""
if not isinstance(columns_to_be_checked, list):
raise ValueError("Please make sure that all your columns passed are strings")
else:
pass
for eachCol in columns_to_be_checked:
if eachCol not in df.columns:
raise KeyError("Please check the column names correspond to values in the DataFrame.")
else:
pass
null_count = 0
for eachColumn in columns_to_be_checked:
prev_null_count = null_count
null_count = prev_null_count + (len(df) - df[eachColumn].count())
return null_count
def check_nat_val(
df: pd.DataFrame,
breakdown_col: str = "Breakdown",
measure_col: str = "Measure",
value_col: str = "Value_Unsuppressed",
nat_val: str = "National"
) -> bool:
"""
Check national value less than or equal to sum of breakdowns.
This function checks that the national value is less than or equal to the
sum of each organisation level breakdown.
This function does not apply to values which are averages.
This function does not apply to values which are percentages calculated
from the numerator and denominator.
Parameters
----------
df : pandas.DataFrame
DataFrame of data to check.
breakdown_col : str, default = "Breakdown"
Column name for the breakdown level.
measure_col : str, default = "Measure"
Column name for measures
value_col : str, default = "Value_Unsuppressed"
Column name for values
nat_val : str, default = "National"
Value in breakdown column denoting national values
Returns
-------
bool
Whether the checks have been passed.
Examples
--------
>>> check_nat_val(
... df = pd.DataFrame({
... "Breakdown" : ['National', 'CCG', 'CCG', 'Provider', 'Provider',
... 'National' ,'CCG', 'CCG', 'Provider', 'Provider','National' ,'CCG', 'CCG',
... 'Provider', 'Provider',],
... "Measure" : ['m1', 'm1', 'm1', 'm1', 'm1', 'm2', 'm2', 'm2', 'm2',
... 'm2', 'm3', 'm3', 'm3', 'm3', 'm3',],
... "Value_Unsuppressed" : [9, 4, 5, 3, 6, 11, 2, 9, 7, 4, 9, 5, 4, 6,
... 3],
... }),
... breakdown_col = "Breakdown",
... measure_col = "Measure",
... value_col = "Value_Unsuppressed",
... nat_val = "National",
... )
True
>>> check_nat_val(
... df = pd.DataFrame({
... "Breakdown" : ['National', 'CCG', 'CCG', 'Provider', 'Provider',
... 'National' ,'CCG', 'CCG', 'Provider', 'Provider','National' ,'CCG', 'CCG',
... 'Provider', 'Provider',],
... "Measure" : ['m1', 'm1', 'm1', 'm1', 'm1', 'm2', 'm2', 'm2', 'm2',
... 'm2', 'm3', 'm3', 'm3', 'm3', 'm3',],
... "Value_Unsuppressed" : [18, 4, 5, 3, 6, 11, 2, 9, 7, 4, 9, 5, 4, 6, 3],
... }),
... breakdown_col = "Breakdown",
... measure_col = "Measure",
... value_col = "Value_Unsuppressed",
... nat_val = "National",
... )
False
"""
if not isinstance(breakdown_col, str) or not isinstance(measure_col, str)\
or not isinstance(value_col, str):
raise ValueError("Please input strings for column indexes.")
if not isinstance(nat_val, str):
raise ValueError("Please input strings for value indexes.")
if breakdown_col not in df.columns or measure_col not in df.columns or\
value_col not in df.columns:
raise KeyError("Check column names correspond to the DataFrame.")
# aggregate values by measure and breakdown
grouped = df.groupby([measure_col, breakdown_col]).agg({value_col: sum})\
.reset_index()
national = grouped.loc[grouped[breakdown_col] == nat_val].reset_index()
non_national = grouped.loc[grouped[breakdown_col] != nat_val].reset_index()
# check values are less than or equal to national value for each measure
join = pd.merge(non_national, national, left_on=measure_col,
right_on=measure_col, how='left')
left = value_col + '_x'
right = value_col + '_y'
join['Check'] = join[right] <= join[left]
result = all(join['Check'])
return result
def check_consistent_measures(
df: pd.DataFrame,
breakdown_col: str = "Org_Level",
measure_col: str = "Measure",
measures_set: set = set(),
) -> bool:
"""
Check every measure is in every geography level.
Parameters
----------
df : pd.DataFrame
DataFrame of data to check.
breakdown_col : str, default = "Org_Level"
Column name for the geography level.
measure_col : str, default = "Measure"
Column name for measure
measures_set : set, default = set()
Set of measures that should be in every geography level. If empty, the existing
global set is presumed to be correct.
Returns
-------
bool
Whether the checks have been passed.
Examples
--------
>>> check_consistent_measures(
... pd.DataFrame({
... "Geog" : ["National" ,"National", "Region", "Region", "Local", "Local",],
... "measure" : ["m1", "m2", "m1", "m2", "m1", "m2",],
... "Value_Unsuppressed" : [4, 2, 2, 1, 2, 1,],
... }),
... breakdown_col = "Geog",
... measure_col = "measure",
... measures_set = set({"m1", "m2"}),
... )
True
>>> check_consistent_measures(
... pd.DataFrame({
... "Org_Level" : ["National" ,"National", "Region", "Region", "Local", "Local",],
... "Measure" : ["m1", "m3", "m1", "m2", "m1", "m2",],
... "Value_Unsuppressed" : [4, 2, 2, 1, 2, 1,],
... })
... )
False
"""
if df.isna().any(axis=None):
raise ValueError(
f"Missing values at locations {list(map(tuple, np.argwhere(df.isna().values)))}"
)
if not isinstance(breakdown_col, str) or not isinstance(measure_col, str):
raise ValueError("Please input strings for column indexes.")
if not isinstance(measures_set, set):
raise ValueError("Please input a set object for measures")
if breakdown_col not in df.columns or measure_col not in df.columns:
raise KeyError("Check column names correspond to the DataFrame.")
# Every geography level should have the same set of measures as the global set.
global_set = measures_set if measures_set else set(df[measure_col].unique())
subsets = df.groupby(breakdown_col) \
.agg({measure_col: "unique"})
subset_agreement = all(set(x) == global_set for x in subsets[measure_col])
return subset_agreement
def check_consistent_submissions(
df: pd.DataFrame,
nat_val: str = "National",
breakdown_col: str = "Org_Level",
submissions_col: str = "Value_Unsuppressed",
measure_col: str = "Measure",
) -> bool:
"""
Check total submissions for each measure are the same across all geography levels
except national.
Parameters
----------
df : pd.DataFrame
DataFrame of data to check.
nat_val : str, default = "National"
Geography level code for national values.
breakdown_col : str, default = "Org_Level"
Column name for the geography level.
submissions_col : str, default = "Value_Unsuppressed"
Column name for the submissions count.
measure_col : str, default = "Measure"
Column name for measure.
Returns
-------
bool
Whether the checks have been passed.
Examples
--------
>>> check_consistent_submissions(
... pd.DataFrame({
... "Geog" : ["N" ,"N", "Region", "Region", "Local", "Local",],
... "measure" : ["m1", "m2", "m1", "m2", "m1", "m2",],
... "submissions" : [4, 2, 2, 1, 2, 1,],
... }),
... nat_val = "N",
... breakdown_col = "Geog",
... submissions_col = "submissions",
... measure_col = "measure",
... )
True
>>> check_consistent_submissions(
... pd.DataFrame({
... "Org_Level" : ["National" ,"National", "Region", "Region", "Local", "Local",],
... "Measure" : ["m1", "m2", "m1", "m2", "m1", "m2",],
... "Value_Unsuppressed" : [4, 2, 3, 1, 2, 1,],
... })
... )
False
"""
if (
not isinstance(submissions_col, str) or
not isinstance(measure_col, str) or
not isinstance(breakdown_col, str) or
not isinstance(nat_val, str)
):
raise ValueError("Please input strings for column names and national geography level.")
if (
submissions_col not in df.columns or
measure_col not in df.columns or
breakdown_col not in df.columns
):
raise KeyError("Check column names correspond to the DataFrame.")
# All non-national measures should have only one unique submission number for each
# geography level.
submissions_by_measure = df[df[breakdown_col] != nat_val] \
.groupby(measure_col) \
.agg({submissions_col: "nunique"})
result = (submissions_by_measure[submissions_col] == 1).all()
return result
def output_checker(
df: pd.DataFrame,
columns_to_be_checked: list,
nat_val: str = "National",
breakdown_col: str = "Breakdown",
value_col: str = "Value_Unsuppressed",
measure_col: str = "Measure",
measures_set: set = set()
) -> bool:
"""
Wrapper to call each of the output checking functions for national figures.
Parameters
----------
df : pandas.DataFrame
DataFrame of data to check.
columns_to_be_checked: list
Given dataframe columns to be checked for null values
nat_val : str, default = "National"
Geography level code for national values.
breakdown_col : str, default = "Breakdown"
Column name for the breakdown level.
value_col : str, default = "Value_Unsuppressed"
Column name for values
measure_col : str, default = "Measure"
Column name for measures
measures_set : set, default = set()
Set of measures that should be in every geography level. If empty, the existing
global set is presumed to be correct.
Returns
----------
True
When the checks have been passed.
Examples
----------
>>> output_checker(
... pd.DataFrame({
... "Breakdown" : ['National', 'CCG', 'CCG', 'Provider', 'Provider',
... 'National' ,'CCG', 'CCG', 'Provider', 'Provider',
... 'National' ,'CCG', 'CCG', 'Provider', 'Provider',],
... "Measure" : ['m1', 'm1', 'm1', 'm1', 'm1',
... 'm2', 'm2', 'm2', 'm2', 'm2',
... 'm3', 'm3', 'm3', 'm3', 'm3',],
... "Value_Unsuppressed" : [20, 10, 10, 10, 10, 8, 4, 4, 4, 4, 12, 6, 6, 6, 6]
... }),
... columns_to_be_checked = ["Breakdown", "Measure", "Value_Unsuppressed"],
... nat_val = "National",
... breakdown_col = "Breakdown",
... measure_col = "Measure",
... measures_set = set({"m1", "m2", "m3"}),
... )
True
"""
assert (check_null(df=df, columns_to_be_checked=columns_to_be_checked) == 0), (
f"Null fields detected at {list(map(tuple, np.argwhere(df.isna().values)))}"
)
assert check_nat_val(
df=df,
breakdown_col=breakdown_col,
measure_col=measure_col,
value_col=value_col,
nat_val=nat_val,
), (
"The national value is more than the sum of breakdowns for one or more measures."
)
assert check_consistent_measures(
df=df,
breakdown_col=breakdown_col,
measure_col=measure_col,
measures_set=measures_set,
), (
"Measures are inconsistent across breakdowns/geography levels."
)
assert check_consistent_submissions(
df=df,
nat_val=nat_val,
breakdown_col=breakdown_col,
submissions_col=value_col,
measure_col=measure_col,
), (
"Submissions are inconsistent for measures across different geography levels"
)
return True