-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathvalidate_data.py
More file actions
325 lines (263 loc) · 10.5 KB
/
Copy pathvalidate_data.py
File metadata and controls
325 lines (263 loc) · 10.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
#!/usr/bin/env python3
"""Validate spreadsheet data before building.
Checks for common issues like missing required fields,
invalid URLs, and missing image files.
"""
import sys
from pathlib import Path
from typing import List, Tuple
from utils import (
load_spreadsheet_all_sheets,
validate_url_format,
check_file_exists,
)
def validate_publications(project_root: Path) -> List[str]:
"""Validate publications.xlsx data.
Returns list of error messages (empty if valid).
Note: Citations are now built from structured fields (authors, year, title, etc.)
rather than a single 'citation' column. Only 'title' is required.
"""
errors = []
xlsx_path = project_root / 'data' / 'publications.xlsx'
if not xlsx_path.exists():
errors.append(f"Missing file: {xlsx_path}")
return errors
try:
data = load_spreadsheet_all_sheets(xlsx_path)
except Exception as e:
errors.append(f"Error loading {xlsx_path}: {e}")
return errors
# Only title is required - citations are built from structured fields
required_fields = ['title']
image_dir = project_root / 'images' / 'publications'
for sheet_name, items in data.items():
for i, item in enumerate(items, 1):
# Check required fields
for field in required_fields:
val = item.get(field, '')
if not val or (isinstance(val, str) and not val.strip()):
errors.append(f"publications/{sheet_name} row {i}: missing {field}")
# Check URL format - allow local file paths (data/pdfs/...) which get
# converted to GitHub URLs by the build script
url = item.get('title_url', '')
if url and not validate_url_format(url) and not url.startswith('data/'):
errors.append(f"publications/{sheet_name} row {i}: invalid URL '{url}'")
# Check image file exists (check_file_exists returns error msg or None)
if item.get('image'):
file_error = check_file_exists(item['image'], image_dir)
if file_error:
errors.append(f"publications/{sheet_name} row {i}: {file_error}")
return errors
def validate_people(project_root: Path) -> List[str]:
"""Validate people.xlsx data.
Returns list of error messages (empty if valid).
"""
errors = []
xlsx_path = project_root / 'data' / 'people.xlsx'
if not xlsx_path.exists():
errors.append(f"Missing file: {xlsx_path}")
return errors
try:
data = load_spreadsheet_all_sheets(xlsx_path)
except Exception as e:
errors.append(f"Error loading {xlsx_path}: {e}")
return errors
image_dir = project_root / 'images' / 'people'
# Validate director
if 'director' in data:
for i, item in enumerate(data['director'], 1):
if not item.get('name'):
errors.append(f"people/director row {i}: missing name")
if item.get('image'):
file_error = check_file_exists(item['image'], image_dir)
if file_error:
errors.append(f"people/director row {i}: {file_error}")
# Validate members
if 'members' in data:
for i, item in enumerate(data['members'], 1):
if not item.get('name'):
errors.append(f"people/members row {i}: missing name")
if item.get('image'):
file_error = check_file_exists(item['image'], image_dir)
if file_error:
errors.append(f"people/members row {i}: {file_error}")
url = item.get('name_url', '')
if url and not validate_url_format(url):
errors.append(f"people/members row {i}: invalid URL '{url}'")
# Validate alumni sheets
for sheet_name in ['alumni_postdocs', 'alumni_grads', 'alumni_managers']:
if sheet_name in data:
for i, item in enumerate(data[sheet_name], 1):
if not item.get('name'):
errors.append(f"people/{sheet_name} row {i}: missing name")
url = item.get('name_url', '')
if url and not validate_url_format(url):
errors.append(f"people/{sheet_name} row {i}: invalid URL '{url}'")
pos_url = item.get('current_position_url', '')
if pos_url and not validate_url_format(pos_url):
errors.append(f"people/{sheet_name} row {i}: invalid position URL '{pos_url}'")
# Validate undergrads
if 'alumni_undergrads' in data:
for i, item in enumerate(data['alumni_undergrads'], 1):
if not item.get('name'):
errors.append(f"people/alumni_undergrads row {i}: missing name")
# Validate collaborators
if 'collaborators' in data:
for i, item in enumerate(data['collaborators'], 1):
if not item.get('name'):
errors.append(f"people/collaborators row {i}: missing name")
url = item.get('url', '')
if url and not validate_url_format(url):
errors.append(f"people/collaborators row {i}: invalid URL '{url}'")
return errors
def validate_software(project_root: Path) -> List[str]:
"""Validate software.xlsx data.
Returns list of error messages (empty if valid).
"""
errors = []
xlsx_path = project_root / 'data' / 'software.xlsx'
if not xlsx_path.exists():
errors.append(f"Missing file: {xlsx_path}")
return errors
try:
data = load_spreadsheet_all_sheets(xlsx_path)
except Exception as e:
errors.append(f"Error loading {xlsx_path}: {e}")
return errors
for sheet_name, items in data.items():
for i, item in enumerate(items, 1):
if not item.get('name'):
errors.append(f"software/{sheet_name} row {i}: missing name")
if not item.get('description'):
errors.append(f"software/{sheet_name} row {i}: missing description")
return errors
def validate_news(project_root: Path) -> List[str]:
"""Validate news.xlsx data.
Returns list of error messages (empty if valid).
"""
errors = []
xlsx_path = project_root / 'data' / 'news.xlsx'
if not xlsx_path.exists():
errors.append(f"Missing file: {xlsx_path}")
return errors
try:
# Load single sheet spreadsheet
import openpyxl
wb = openpyxl.load_workbook(xlsx_path, read_only=True, data_only=True)
sheet = wb.active
headers = [cell.value for cell in sheet[1]]
items = []
for row in sheet.iter_rows(min_row=2, values_only=True):
if not any(cell is not None for cell in row):
continue
row_dict = {}
for header, value in zip(headers, row):
row_dict[header] = value if value is not None else ''
items.append(row_dict)
wb.close()
except Exception as e:
errors.append(f"Error loading {xlsx_path}: {e}")
return errors
image_dir = project_root / 'images' / 'news'
for i, item in enumerate(items, 1):
# Check required fields
if not item.get('title'):
errors.append(f"news row {i}: missing title")
if not item.get('content'):
errors.append(f"news row {i}: missing content")
# Check URL format if title_url is provided
# Allow local file paths (like people.html) and data/ paths
url = item.get('title_url', '')
if url and not validate_url_format(url) and not url.endswith('.html') and not url.startswith('data/'):
errors.append(f"news row {i}: invalid URL '{url}'")
# Check image file exists
if item.get('image'):
file_error = check_file_exists(item['image'], image_dir)
if file_error:
errors.append(f"news row {i}: {file_error}")
# Validate date format if provided
date_val = item.get('date', '')
if date_val:
import re
date_str = str(date_val)
if not re.match(r'^\d{4}-\d{2}-\d{2}$', date_str):
errors.append(f"news row {i}: invalid date format '{date_str}' (expected YYYY-MM-DD)")
return errors
def validate_templates(project_root: Path) -> List[str]:
"""Validate that all required templates exist.
Returns list of error messages (empty if valid).
"""
errors = []
templates_dir = project_root / 'templates'
required_templates = [
'publications.html',
'people.html',
'software.html',
'news.html'
]
for template in required_templates:
template_path = templates_dir / template
if not template_path.exists():
errors.append(f"Missing template: {template_path}")
return errors
def main():
"""Run all validations and report results."""
project_root = Path(__file__).parent.parent
print("Validating data files...")
print("=" * 50)
all_errors = []
# Validate templates first
template_errors = validate_templates(project_root)
if template_errors:
print("\nTemplate errors:")
for error in template_errors:
print(f" - {error}")
all_errors.extend(template_errors)
else:
print("Templates: OK")
# Validate publications
pub_errors = validate_publications(project_root)
if pub_errors:
print("\nPublications errors:")
for error in pub_errors:
print(f" - {error}")
all_errors.extend(pub_errors)
else:
print("Publications: OK")
# Validate people
people_errors = validate_people(project_root)
if people_errors:
print("\nPeople errors:")
for error in people_errors:
print(f" - {error}")
all_errors.extend(people_errors)
else:
print("People: OK")
# Validate software
sw_errors = validate_software(project_root)
if sw_errors:
print("\nSoftware errors:")
for error in sw_errors:
print(f" - {error}")
all_errors.extend(sw_errors)
else:
print("Software: OK")
# Validate news
news_errors = validate_news(project_root)
if news_errors:
print("\nNews errors:")
for error in news_errors:
print(f" - {error}")
all_errors.extend(news_errors)
else:
print("News: OK")
# Summary
print("\n" + "=" * 50)
if all_errors:
print(f"Validation completed with {len(all_errors)} error(s)")
sys.exit(1)
else:
print("Validation completed successfully!")
sys.exit(0)
if __name__ == '__main__':
main()