-
Notifications
You must be signed in to change notification settings - Fork 9
Expand file tree
/
Copy pathreport.ts
More file actions
114 lines (106 loc) · 3.51 KB
/
Copy pathreport.ts
File metadata and controls
114 lines (106 loc) · 3.51 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
/**
* Run the harness over a set of movements and produce a scorecard.
* Pure data-in/data-out so it can drive a CLI, CI gate, or LLM benchmark.
*/
import { probeMovement } from "./probe.js";
import { genericChecks, MOVEMENT_CHECKS, type CheckOutcome } from "./checks.js";
import type { Character, Proportions } from "posecode-render";
import type { ClipDiagnostics } from "./diagnostics.js";
export interface MovementSource {
/** Identifier matched against MOVEMENT_CHECKS (e.g. file stem "deadlift"). */
movement: string;
source: string;
}
export interface MovementReport {
movement: string;
parseOk: boolean;
clampWarnings: number;
checks: CheckOutcome[];
/** Strict clip-wide constraint diagnostics; warnings do not change CI pass counts. */
diagnostics: ClipDiagnostics;
passed: number;
total: number;
}
export interface EvalReport {
movements: MovementReport[];
summary: {
movements: number;
parseFailures: number;
clampWarnings: number;
checksPassed: number;
checksTotal: number;
constraintWarnings: number;
};
}
export interface EvalOptions {
/** Driver proportions used by the production character being evaluated. */
proportions?: Proportions;
/** Optional retargeted visible character sampled after each solved phase. */
character?: Character;
/** Sampling rate for clip-wide grounding/collision diagnostics. Defaults to 12Hz. */
diagnosticSampleRateHz?: number;
}
export function runEval(
sources: readonly MovementSource[],
options: EvalOptions = {},
): EvalReport {
const movements = sources.map((s) => evalMovement(s, options));
return {
movements,
summary: {
movements: movements.length,
parseFailures: movements.filter((m) => !m.parseOk).length,
clampWarnings: movements.reduce((n, m) => n + m.clampWarnings, 0),
checksPassed: movements.reduce((n, m) => n + m.passed, 0),
checksTotal: movements.reduce((n, m) => n + m.total, 0),
constraintWarnings: movements.reduce((n, m) => n + m.diagnostics.warnings.length, 0),
},
};
}
function evalMovement(
{ movement, source }: MovementSource,
options: EvalOptions,
): MovementReport {
const result = probeMovement(
source,
options.proportions,
options.character,
{ diagnosticSampleRateHz: options.diagnosticSampleRateHz },
);
const specific = MOVEMENT_CHECKS.find((m) => m.movement === movement);
const checks = [
...genericChecks(result),
...(specific?.checks.map((c) => c(result)) ?? []),
];
return {
movement,
parseOk: result.ok,
clampWarnings: result.warnings.length,
checks,
diagnostics: result.diagnostics,
passed: checks.filter((c) => c.pass).length,
total: checks.length,
};
}
/** Render a compact human-readable scorecard. */
export function renderReport(report: EvalReport): string {
const lines: string[] = [];
for (const m of report.movements) {
const mark = m.passed === m.total ? "✓" : "✗";
lines.push(`${mark} ${m.movement} ${m.passed}/${m.total}`);
for (const c of m.checks.filter((c) => !c.pass)) {
lines.push(` ✗ ${c.id}: ${c.detail}`);
}
for (const warning of m.diagnostics.warnings) {
lines.push(` ⚠ ${warning.id}: ${warning.detail}`);
}
}
const s = report.summary;
lines.push("");
lines.push(
`${s.checksPassed}/${s.checksTotal} checks · ${s.movements} movements · ` +
`${s.parseFailures} parse failures · ${s.clampWarnings} clamp warnings · ` +
`${s.constraintWarnings} constraint warnings`,
);
return lines.join("\n");
}