forked from inventree/InvenTree
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcheck_exported_data.py
More file actions
238 lines (194 loc) · 8.49 KB
/
Copy pathcheck_exported_data.py
File metadata and controls
238 lines (194 loc) · 8.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
"""Script to check a data file exported using the 'export-records' command.
This script is intended to be used as part of the CI workflow,
in conjunction with the "workflows/import_export.yaml" workflow.
In reads the exported data file, to ensure that:
- The file can be read and parsed as JSON
- The file contains the expected metadata
- The file contains the expected plugin configuration
- The file contains the expected plugin database records
It can also optionally check the presence / absence of several categories of
data which 'export-records' can include or exclude via --include-x /
--exclude-x flags (email logs, API tokens, SSO app/token data, user sessions,
and non-empty group/user permissions) - pass e.g. '--check-email include' or
'--check-email exclude' to assert that category was (or was not) found in the
exported data. Any '--check-x' option which is *not* passed is simply not
checked at all (not even implicitly assumed absent) - so existing invocations
which don't pass any of them keep working unchanged.
"""
PLUGIN_KEY = 'dummy_app_plugin'
PLUGIN_SLUG = 'dummy-app-plugin'
import argparse
import json
import os
def check_category(
data: list[dict], label: str, model_names: list[str], expect: str | None
):
"""Check that all of the given model names are present / absent as expected.
Arguments:
data: The loaded (parsed) exported data file.
label: A human-readable label for this category, for error messages.
model_names: The Django model labels (e.g. 'common.emailmessage')
which make up this category.
expect: 'include' - at least one entry for *each* model name must be
present. 'exclude' - *no* entry for *any* model name may be
present. None - this category was not requested to be checked;
do nothing.
"""
if expect is None:
return
expect_present = expect == 'include'
counts = dict.fromkeys(model_names, 0)
for entry in data:
model = entry.get('model', None)
if model in counts:
counts[model] += 1
if expect_present:
for model, count in counts.items():
if count == 0:
print(f"Error: Expected '{label}' data ('{model}') was not found")
exit(1)
print(f"Found expected '{label}' data ({counts})")
else:
for model, count in counts.items():
if count > 0:
print(
f"Error: '{label}' data ('{model}') was found, but should have been excluded ({count} record(s))"
)
exit(1)
print(f"Confirmed '{label}' data was correctly excluded")
def check_permissions(data: list[dict], expect: str | None):
"""Check that auth.group / auth.user permission fields are stripped or preserved as expected.
Arguments:
data: The loaded (parsed) exported data file.
expect: 'include' - at least one auth.group / auth.user entry must
have non-empty permissions. 'exclude' - all such entries must have
empty permissions. None - not checked; do nothing.
"""
if expect is None:
return
expect_present = expect == 'include'
group_perms = [
entry['fields'].get('permissions', [])
for entry in data
if entry.get('model') == 'auth.group'
]
user_perms = [
entry['fields'].get('user_permissions', [])
for entry in data
if entry.get('model') == 'auth.user'
]
any_group_perms = any(group_perms)
any_user_perms = any(user_perms)
if expect_present:
if not any_group_perms and not any_user_perms:
print(
'Error: Expected at least one auth.group / auth.user entry with '
'non-empty permissions, but all were empty'
)
exit(1)
print('Found expected non-empty group/user permissions')
else:
if any_group_perms or any_user_perms:
print(
'Error: Found non-empty group/user permissions, but they should '
'have been stripped'
)
exit(1)
print('Confirmed group/user permissions were correctly stripped')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Check exported data file')
parser.add_argument('datafile', help='Path to the exported data file (JSON)')
# Plugin data is checked unconditionally below (it always has been) - this
# just controls which direction is expected, mirroring export-records' own
# --exclude-plugins flag (plugin data is included by default).
parser.add_argument('--exclude-plugins', action='store_true')
# The remaining categories are only checked when explicitly requested -
# pass 'include' or 'exclude' to assert that direction, or omit the flag
# entirely to skip checking that category (the default, for backwards
# compatibility with existing invocations that don't pass any of these).
for flag in ('email', 'tokens', 'sso', 'session', 'permissions'):
parser.add_argument(f'--check-{flag}', choices=['include', 'exclude'])
args = parser.parse_args()
if not os.path.isfile(args.datafile):
print(f'Error: File not found: {args.datafile}')
exit(1)
with open(args.datafile, encoding='utf-8') as f:
try:
data = json.load(f)
print(f'Successfully loaded data from {args.datafile}')
print(f'Number of records: {len(data)}')
except json.JSONDecodeError as e:
print(f'Error: Failed to parse JSON file: {e}')
exit(1)
found_metadata = False
found_installed_apps = False
found_plugin_config = False
plugin_data_records = {}
# Inspect the data and check that it has the expected structure and content.
for entry in data:
# Check metadata entry for expected values
if entry.get('metadata', False):
print('Found metadata entry')
found_metadata = True
expected_apps = ['InvenTree', 'allauth', 'dbbackup', PLUGIN_KEY]
apps = entry.get('installed_apps', [])
for app in expected_apps:
if app not in apps:
print(f'- Expected app "{app}" not found in installed apps list')
exit(1)
found_installed_apps = True
elif entry.get('model', None) == 'plugin.pluginconfig':
key = entry['fields']['key']
if key == PLUGIN_SLUG:
print(f'Found plugin configuration for plugin "{PLUGIN_KEY}"')
found_plugin_config = True
elif entry.get('model', None) == f'{PLUGIN_KEY}.examplemodel':
key = entry['fields']['key']
value = entry['fields']['value']
plugin_data_records[key] = value
if not found_metadata:
print('Error: No metadata entry found in exported data')
exit(1)
if not found_installed_apps:
print(
f'Error: Plugin "{PLUGIN_KEY}" not found in installed apps list in metadata'
)
exit(1)
# Plugin data is included by default (export-records only excludes it when
# given --exclude-plugins), so preserve that as the default expectation here
if not args.exclude_plugins:
if not found_plugin_config:
print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
exit(1)
# Check the extracted plugin records
expected_keys = ['alpha', 'beta', 'gamma', 'delta']
for key in expected_keys:
if key not in plugin_data_records:
print(
f'Error: Expected plugin record with key "{key}" not found in exported data'
)
exit(1)
elif found_plugin_config:
print('Error: Plugin data was found, but should have been excluded')
exit(1)
else:
print('Confirmed plugin data was correctly excluded')
# Content-excludes checks - only run for '--check-x' flags that were actually passed
check_category(
data, 'email', ['common.emailmessage', 'common.emailthread'], args.check_email
)
check_category(data, 'tokens', ['users.apitoken'], args.check_tokens)
check_category(
data,
'sso',
['socialaccount.socialapp', 'socialaccount.socialtoken'],
args.check_sso,
)
check_category(
data,
'session',
['sessions.session', 'usersessions.usersession'],
args.check_session,
)
check_permissions(data, args.check_permissions)
print('All checks passed successfully!')