-
-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Expand file tree
/
Copy pathverify.py
More file actions
251 lines (223 loc) · 10.7 KB
/
Copy pathverify.py
File metadata and controls
251 lines (223 loc) · 10.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
#!/usr/bin/env python3
"""Run the course contract against starter or solution.
Objective completion evidence for "Data visualization" (da-visualization).
``progress`` prints checkpoint claim codes for recording on flypython.com.
Dependencies (pandas, matplotlib) managed with uv — run ``uv sync`` inside
this folder first.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import importlib.util
import json
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent
# Optional shared claim-receipt producer (docs/CLAIM-RECEIPT.md). The course
# folder still verifies standalone — without the tools/ sibling or without
# FLYPYTHON_CLAIM_SECRET set, verify.py behaves exactly as before.
try:
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools"))
import claim_receipt
except ImportError:
claim_receipt = None
EXPECTED_STARTER_FAILURES = (
"test_load_returns_rows",
"test_region_totals",
"test_daily_count_and_total",
"test_three_pngs_exist_and_are_valid",
"test_summary_json_written",
)
COURSE_ID = 'course-da-visualization'
# Documented constant: claim codes derive deterministically from
# (COURSE_ID, checkpoint_id, COURSE_SALT). Spot-checkable self-reported
# evidence, not tamper-proof secrets — see docs/repo-plan-0.0.4.md FP-411.
COURSE_SALT = '7c1f4a2e9b8d6f03'
CHECKPOINTS = [
{"id": "l01", "gate": "attest", "title": 'A chart is packaged evidence'},
{"id": "l02", "gate": "attest", "title": 'Spec first, chart second'},
{"id": "l03", "gate": "starter-suite", "title": 'Three charts and a summary'},
{"id": "l04", "gate": "both-suites", "title": 'Numbers match the charts'},
{"id": "l05", "gate": "attest", "title": 'What verification cannot see'},
]
def _deps_available():
return (importlib.util.find_spec("pandas") is not None
and importlib.util.find_spec("matplotlib") is not None)
def _deps_hint():
print(
"pandas/matplotlib not installed. Run `uv sync` (or "
"`pip install -r requirements.txt`) inside this course folder first.",
file=sys.stderr,
)
def _claim_code(checkpoint_id):
digest = hashlib.sha256(
(COURSE_ID + ":" + checkpoint_id + ":" + COURSE_SALT).encode("utf-8")
).digest()
return base64.b32encode(digest).decode("ascii")[:8]
def _display_title(title):
# FP-820: shared-core checkpoint titles are stored "Chinese / English";
# the default command prints English first, Chinese after (FP-709 debt).
if " / " in title:
left, _, right = title.partition(" / ")
if any("\u4e00" <= character <= "\u9fff" for character in left):
return right + " / " + left
return title
def _run_suite(implementation):
environment = os.environ.copy()
environment["PYTHONPATH"] = str(ROOT / implementation)
return subprocess.run(
[sys.executable, "-m", "unittest", "discover", "-s", str(ROOT / "tests")],
env=environment, check=False, capture_output=True, text=True,
)
def run_progress(as_json, receipt_out=None, default=False):
starter = solution = None
starter_ms = solution_ms = 0
if not _deps_available():
_deps_hint()
starter_ok = solution_ok = False
else:
import time
t0 = time.monotonic()
starter = _run_suite("starter")
starter_ms = int((time.monotonic() - t0) * 1000)
t0 = time.monotonic()
solution = _run_suite("solution")
solution_ms = int((time.monotonic() - t0) * 1000)
starter_ok = starter.returncode == 0
solution_ok = solution.returncode == 0
rows = []
for checkpoint in CHECKPOINTS:
gate = checkpoint["gate"]
if gate == "attest":
status, kind = "attest", "attested"
elif gate == "starter-suite":
status = "passed" if starter_ok else "open"
kind = "objective"
else:
status = "passed" if (starter_ok and solution_ok) else "open"
kind = "objective"
code = _claim_code(checkpoint["id"]) if status in ("passed", "attest") else None
row = dict(checkpoint)
row["status"] = status
row["kind"] = kind
row["claim_code"] = code
rows.append(row)
secret = claim_receipt.receipts_enabled() if claim_receipt else None
receipts = []
if secret:
starter_tests = claim_receipt.count_tests(starter.stderr) if starter else 0
solution_tests = claim_receipt.count_tests(solution.stderr) if solution else 0
solution_hash = claim_receipt.solution_sha256(ROOT)
for checkpoint in CHECKPOINTS:
gate = checkpoint["gate"]
if gate == "starter-suite":
passed, tests, ms = starter_ok, starter_tests, starter_ms
elif gate == "both-suites":
passed = starter_ok and solution_ok
tests, ms = starter_tests + solution_tests, starter_ms + solution_ms
else:
continue
receipts.append(claim_receipt.make_receipt(
COURSE_ID, checkpoint["id"], passed=passed, tests=tests,
duration_ms=ms, impl_dir=ROOT / "starter",
solution_hash=solution_hash, secret=secret))
if default and not as_json:
# FP-820: bare ``python verify.py`` — check the learner's own
# implementation, print per-checkpoint status and earned claim codes,
# English first then Chinese (FP-709 debt). ``progress`` keeps its
# exact published output; this block is the only new surface.
print("Course " + COURSE_ID)
print("Suites: starter " + ("passed" if starter_ok else "not passed")
+ " / solution " + ("passed" if solution_ok else "not passed"))
for row in rows:
state = row["status"] + (" (self-attested)" if row["kind"] == "attested" else "")
code = row["claim_code"] if row["claim_code"] else "\u2014"
print(" " + row["id"] + " " + _display_title(row["title"]) + " [" + state + "] " + code)
open_gates = [row["id"] for row in rows
if row["kind"] == "objective" and row["status"] != "passed"]
if open_gates:
print("Next: keep implementing starter/ until " + ", ".join(open_gates)
+ " show [passed]; then re-run: python verify.py")
else:
print("All objective checkpoints passed — the codes above are ready"
" to submit (batch POST /api/claims, see SKILL.md §5).")
print("\u4e2d\u6587\uff1a\u68c0\u67e5\u70b9\u72b6\u6001\u4e0e\u8ba4\u9886\u7801\u89c1\u4e0a\uff1b\u5e26 [passed]/[attest] \u7684\u884c\u6709\u8ba4\u9886\u7801\uff0c[open] \u7684\u884c\u7ee7\u7eed\u5728 starter/ \u91cc\u5b9e\u73b0\u540e\u518d\u8dd1\u3002")
print("Claim codes are self-reported evidence, recorded at flypython.com; never a certificate.")
print("\u8ba4\u9886\u7801\u662f\u81ea\u6211\u62a5\u544a\u7684\u8bc1\u636e\uff0c\u8bb0\u5f55\u5728 flypython.com\uff1b\u4e0d\u662f\u8bc1\u4e66\u3002")
if secret:
print(f"Signed run receipts prepared for {len(receipts)} gated checkpoint(s);"
" submit each with its claim to mark it as a local-run receipt.")
return 1 if open_gates else 0
if as_json:
document = {"course": COURSE_ID,
"starter_suite_passed": starter_ok,
"solution_suite_passed": solution_ok,
"checkpoints": rows}
if secret:
document["receipts"] = receipts
print(json.dumps(document, ensure_ascii=False, indent=2))
else:
starter_state = "passed" if starter_ok else "not passed"
solution_state = "passed" if solution_ok else "not passed"
print("Course " + COURSE_ID)
print("Suites: starter " + starter_state + " / solution " + solution_state)
for row in rows:
state = row["status"] + (" (self-attested)" if row["kind"] == "attested" else "")
code = "claim code " + row["claim_code"] if row["claim_code"] else "—"
print(" " + row["id"] + " " + row["title"] + " [" + state + "] " + code)
print("Claim codes are self-reported evidence, recorded at flypython.com; never a certificate.")
if secret:
print(f"Signed run receipts prepared for {len(receipts)} gated checkpoint(s);"
" submit each with its claim to mark them as a local-run receipt.")
if receipt_out and secret:
Path(receipt_out).write_text(
json.dumps({"receipts": receipts}, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8")
print(f"Wrote {len(receipts)} receipt(s) to {receipt_out}", file=sys.stderr)
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("implementation", choices=("progress", "starter", "solution"),
nargs="?", default=None)
parser.add_argument("--json", action="store_true")
parser.add_argument("--expect-failure", action="store_true")
parser.add_argument("--receipt-out", metavar="PATH", help="write signed run receipts JSON (requires FLYPYTHON_CLAIM_SECRET)")
args = parser.parse_args()
if args.implementation in (None, "progress"):
# FP-820: the bare command is the learner's default — same engine as
# ``progress``, English-first output, exit 1 while gates stay open.
# The four explicit usages remain for maintainers.
return run_progress(args.json, args.receipt_out,
default=args.implementation is None)
if not _deps_available():
_deps_hint()
return 1
environment = os.environ.copy()
environment["PYTHONPATH"] = str(ROOT / args.implementation)
result = subprocess.run(
[sys.executable, "-m", "unittest", "discover", "-s", str(ROOT / "tests")],
env=environment, check=False, capture_output=True, text=True,
)
if args.expect_failure:
if result.returncode == 0:
print("Expected the starter to fail, but it passed.", file=sys.stderr)
return 1
output = (result.stdout or "") + (result.stderr or "")
missing = [n for n in EXPECTED_STARTER_FAILURES if n not in output]
if missing:
print("Starter failed for unexpected reasons:", file=sys.stderr)
print("\n".join(missing), file=sys.stderr)
print(output, file=sys.stderr)
return 1
print("Expected starter state reproduced: chart functions unimplemented.")
return 0
if result.returncode == 0:
print(f"{args.implementation}: all tests passed")
else:
sys.stderr.write((result.stderr or "") or (result.stdout or ""))
return result.returncode
if __name__ == "__main__":
raise SystemExit(main())