Repository navigation
Expand file tree
/
Copy pathcli.py
More file actions
98 lines (71 loc) · 2.43 KB
/
Copy pathcli.py
File metadata and controls
98 lines (71 loc) · 2.43 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
"""Command-line interface for docsgraph-eval.
Provides one ``run`` subcommand per benchmark area, plus a ``run all``
command that runs every area. Each command currently reports that the
corresponding benchmark is not yet implemented.
"""
import typer
app = typer.Typer(
name="docsgraph-eval",
help=(
"Independent benchmark and testing suite for docsgraph: ocr, "
"extraction, retrieval, evidence_attribution, graph_generation, "
"permissions, sync, and offline_consistency."
),
)
run_app = typer.Typer(help="Run a benchmark for a specific area, or `all` areas.")
app.add_typer(run_app, name="run")
BENCHMARK_AREAS: tuple[str, ...] = (
"ocr",
"extraction",
"retrieval",
"evidence_attribution",
"graph_generation",
"permissions",
"sync",
"offline_consistency",
)
def _not_implemented(area: str) -> None:
"""Report that a benchmark area has no runnable implementation yet.
Args:
area: Name of the benchmark area.
"""
typer.echo(f"[{area}] not yet implemented")
@run_app.command("ocr")
def run_ocr() -> None:
"""Run the OCR accuracy benchmark."""
_not_implemented("ocr")
@run_app.command("extraction")
def run_extraction() -> None:
"""Run the structured extraction benchmark."""
_not_implemented("extraction")
@run_app.command("retrieval")
def run_retrieval() -> None:
"""Run the retrieval quality benchmark."""
_not_implemented("retrieval")
@run_app.command("evidence-attribution")
def run_evidence_attribution() -> None:
"""Run the evidence attribution benchmark."""
_not_implemented("evidence_attribution")
@run_app.command("graph-generation")
def run_graph_generation() -> None:
"""Run the knowledge graph generation quality benchmark."""
_not_implemented("graph_generation")
@run_app.command("permissions")
def run_permissions() -> None:
"""Run the permissions/access-control benchmark."""
_not_implemented("permissions")
@run_app.command("sync")
def run_sync() -> None:
"""Run the sync protocol convergence benchmark."""
_not_implemented("sync")
@run_app.command("offline-consistency")
def run_offline_consistency() -> None:
"""Run the offline/online consistency benchmark."""
_not_implemented("offline_consistency")
@run_app.command("all")
def run_all() -> None:
"""Run every benchmark area."""
for area in BENCHMARK_AREAS:
_not_implemented(area)
if __name__ == "__main__":
app()