-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathstage2_plan_diff.py
More file actions
132 lines (115 loc) · 5.04 KB
/
Copy pathstage2_plan_diff.py
File metadata and controls
132 lines (115 loc) · 5.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
#!/usr/bin/env python3
"""Plan-level diff of the lexical planner across two trees, over the two question sets.
The Stage 2 escalation council asked for the check the record had only sketched:
"compare old/new plans for BOTH complete query sets, not selected textual patterns."
``dump`` writes, for every eligible census question (the full population the census
receipt scored) and every gold DEV item, exactly what :func:`retrieval.plan_query`
returns under the planner of the tree it runs in; ``compare`` lists every id whose
plan differs between two dumps. Run ``dump`` once per tree (a worktree at the older
commit, then HEAD) and ``compare`` the two files. Read-only against the DB.
uv run python scripts/eval/stage2_plan_diff.py dump --db <sessions.db> --out A.json
uv run python scripts/eval/stage2_plan_diff.py compare A.json B.json --out receipt.json
"""
from __future__ import annotations
import argparse
import hashlib
import json
import sqlite3
import subprocess
from dataclasses import asdict
from pathlib import Path
def _git(*args: str) -> str:
return subprocess.run(["git", *args], capture_output=True, text=True, check=True).stdout.strip()
def dump(db: Path, out: Path) -> None:
from agent_session_tools import retrieval
from agent_session_tools.eval.census import collect_questions
from agent_session_tools.eval.gold import load_gold
conn = sqlite3.connect(db.resolve().as_uri() + "?mode=ro", uri=True)
conn.row_factory = sqlite3.Row
try:
questions = collect_questions(conn)
finally:
conn.close()
gold = load_gold()
planner_path = Path(retrieval.__file__)
planner_source = planner_path.read_bytes()
tree = str(planner_path.parent) # the tree the planner was imported from, not the caller's
payload = {
"schema": "studyloop.plan-dump/v1",
"planner_path": str(planner_path),
"git_commit": _git("-C", tree, "rev-parse", "HEAD"),
"tree_dirty": bool(_git("-C", tree, "status", "--porcelain", "--", ".")),
"planner_sha256": hashlib.sha256(planner_source).hexdigest(),
"db": str(db),
"census": {
q.message_id: {"text": q.text, "plan": asdict(retrieval.plan_query(q.text))}
for q in questions
},
"gold": {
item["id"]: {
"text": item["question"],
"plan": asdict(retrieval.plan_query(item["question"])),
}
for item in gold.items
},
}
out.write_text(json.dumps(payload, indent=1, sort_keys=True) + "\n", encoding="utf-8")
print(f"census {len(payload['census'])} · gold {len(payload['gold'])} → {out}")
def _shape(plan: dict) -> dict:
return {
"explicit": plan["explicit"],
"n_terms": len(plan["terms"]),
"n_queries": len(plan["queries"]),
"note": plan.get("note"),
}
def compare(before: Path, after: Path, out: Path) -> None:
a = json.loads(before.read_text(encoding="utf-8"))
b = json.loads(after.read_text(encoding="utf-8"))
receipt: dict = {
"schema": "studyloop.plan-diff/v1",
"before": {k: a[k] for k in ("git_commit", "tree_dirty", "planner_sha256")},
"after": {k: b[k] for k in ("git_commit", "tree_dirty", "planner_sha256")},
"sets": {},
}
for name in ("census", "gold"):
ids_a, ids_b = set(a[name]), set(b[name])
common = sorted(ids_a & ids_b)
changed = [i for i in common if a[name][i]["plan"] != b[name][i]["plan"]]
receipt["sets"][name] = {
"n_before": len(ids_a),
"n_after": len(ids_b),
"n_common": len(common),
"n_changed": len(changed),
# Ids, a hash of the text and the plan SHAPE only: learner turns are
# pasted material and may carry credentials, so no text and no terms
# leave the dump files (which stay outside the repository).
"changed": [
{
"id": i,
"text_sha256": hashlib.sha256(b[name][i]["text"].encode()).hexdigest(),
"before": _shape(a[name][i]["plan"]),
"after": _shape(b[name][i]["plan"]),
}
for i in changed
],
}
out.write_text(json.dumps(receipt, indent=1, sort_keys=True) + "\n", encoding="utf-8")
for name, s in receipt["sets"].items():
print(f"{name}: {s['n_changed']} of {s['n_common']} plans changed")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
sub = parser.add_subparsers(dest="cmd", required=True)
d = sub.add_parser("dump")
d.add_argument("--db", type=Path, required=True)
d.add_argument("--out", type=Path, required=True)
c = sub.add_parser("compare")
c.add_argument("before", type=Path)
c.add_argument("after", type=Path)
c.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
if args.cmd == "dump":
dump(args.db, args.out)
else:
compare(args.before, args.after, args.out)
if __name__ == "__main__":
main()