forked from eval-protocol/python-sdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathplugin.py
More file actions
420 lines (368 loc) · 14.8 KB
/
Copy pathplugin.py
File metadata and controls
420 lines (368 loc) · 14.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
"""
Pytest plugin for Eval Protocol developer ergonomics.
Adds a discoverable CLI flag `--ep-max-rows` to control how many rows
evaluation_test processes. This sets the environment variable
`EP_MAX_DATASET_ROWS` so the core decorator can apply it uniformly to
both URL datasets and in-memory input_messages.
Usage:
- CLI: pytest --ep-max-rows=2 # or --ep-max-rows=all for no limit
- Defaults: If not provided, no override is applied (tests use the
max_dataset_rows value set in the decorator).
"""
import logging
import os
from typing import Optional
import json
import pathlib
import sys
from pytest import StashKey
import pytest
def pytest_addoption(parser) -> None:
group = parser.getgroup("eval-protocol")
group.addoption(
"--ep-max-rows",
action="store",
default=None,
help=(
"Limit number of dataset rows processed by evaluation_test. "
"Pass an integer (e.g., 2, 50) or 'all' for no limit."
),
)
group.addoption(
"--ep-num-runs",
action="store",
default=None,
help=("Override the number of runs for evaluation_test. Pass an integer (e.g., 1, 5, 10)."),
)
group.addoption(
"--ep-max-concurrent-rollouts",
action="store",
default=None,
help=("Override the maximum number of concurrent rollouts. Pass an integer (e.g., 8, 50, 100)."),
)
group.addoption(
"--ep-max-concurrent-evaluations",
action="store",
default=None,
help=("Override the maximum number of concurrent evaluations. Pass an integer (e.g., 8, 50, 100)."),
)
group.addoption(
"--ep-print-summary",
action="store_true",
default=False,
help=("Print a concise summary line (suite/model/effort/agg score) at the end of each evaluation_test."),
)
group.addoption(
"--ep-summary-json",
action="store",
default=None,
help=("Write a JSON summary artifact at the given path (e.g., ./outputs/aime_low.json)."),
)
# deprecate this later
group.addoption(
"--ep-input-param",
action="append",
default=None,
help=(
"Override rollout input parameters. Can be used multiple times. "
"Format: key=value or JSON via @path.json. Examples: "
"--ep-input-param temperature=0 --ep-input-param @params.json"
),
)
group.addoption(
"--ep-reasoning-effort",
action="store",
default=None,
help=(
"Set reasoning.effort for providers that support it (e.g., Fireworks) via LiteLLM extra_body. "
"Values: low|medium|high|none"
),
)
group.addoption(
"--ep-max-retry",
action="store",
default=None,
help=("Failed rollouts (with rollout_status.code indicating error) will be retried up to this many times."),
)
group.addoption(
"--ep-fail-on-max-retry",
action="store",
default="true",
choices=["true", "false"],
help=(
"Whether to fail the entire rollout when permanent failures occur after max retries. "
"Default: true (fail on permanent failures). Set to 'false' to continue with remaining rollouts."
),
)
group.addoption(
"--ep-success-threshold",
action="store",
default=None,
help=("Override the success threshold for evaluation_test. Pass any float (e.g., 0.8 or -1e9 to disable)."),
)
group.addoption(
"--ep-se-threshold",
action="store",
default=None,
help=(
"Override the standard error threshold for evaluation_test. "
"Pass a float >= 0.0 (e.g., 0.05). If only this is set, success threshold defaults to 0.0."
),
)
group.addoption(
"--ep-no-upload",
action="store_true",
default=False,
help=(
"Disable saving and uploading of detailed experiment JSON files to Fireworks. "
"Default: false (experiment JSONs are saved and uploaded by default)."
),
)
group.addoption(
"--ep-jsonl-path",
default=None,
help=("Load input from a jsonl file that is already in EvaluationRow or openai CHAT format"),
)
group.addoption(
"--ep-completion-params",
default=[],
action="append",
help=("Overwrite completion params with json. Can be used multiple times. "),
)
group.addoption(
"--ep-remote-rollout-processor-base-url",
default=None,
help=("If set, use this base URL for remote rollout processing. Example: http://localhost:8000"),
)
group.addoption(
"--ep-no-op-rollout-processor",
action="store_true",
default=False,
help=(
"Override the rollout processor to use NoOpRolloutProcessor, which passes input dataset through unchanged."
),
)
group.addoption(
"--ep-output-dir",
default=None,
help=("If set, save evaluation results to this directory in jsonl format."),
)
def _normalize_max_rows(val: Optional[str]) -> Optional[str]:
if val is None:
return None
s = val.strip().lower()
if s == "all":
return "None"
# Validate int; if invalid, ignore and return None (no override)
try:
int(s)
return s
except ValueError:
return None
def _normalize_number(val: Optional[str]) -> Optional[str]:
if val is None:
return None
s = val.strip()
# Validate int; if invalid, ignore and return None (no override)
try:
num = int(s)
if num <= 0:
return None # num_runs must be positive
return str(num)
except ValueError:
return None
def _normalize_success_threshold(val: Optional[str]) -> Optional[float]:
"""Normalize success threshold value as float."""
if val is None:
return None
try:
return float(val.strip())
except ValueError:
return None
def _normalize_se_threshold(val: Optional[str]) -> Optional[float]:
"""Normalize standard error threshold value as float >= 0.0."""
if val is None:
return None
try:
threshold_float = float(val.strip())
if threshold_float >= 0.0:
return threshold_float
else:
return None # standard error must be >= 0
except ValueError:
return None
def _build_passed_threshold_env(success: Optional[float], se: Optional[float]) -> Optional[str]:
"""Build the EP_PASSED_THRESHOLD environment variable value from the two separate thresholds."""
if success is None and se is None:
return None
if se is None:
return str(success)
else:
success_val = success if success is not None else 0.0
threshold_dict = {"success": success_val, "standard_error": se}
return json.dumps(threshold_dict)
def pytest_configure(config) -> None:
# Quiet LiteLLM INFO spam early in pytest session unless user set a level
try:
if os.environ.get("LITELLM_LOG") is None:
os.environ["LITELLM_LOG"] = "ERROR"
_llog = logging.getLogger("LiteLLM")
_llog.setLevel(logging.CRITICAL)
_llog.propagate = False
for _h in list(_llog.handlers):
_llog.removeHandler(_h)
except Exception:
pass
cli_val = config.getoption("--ep-max-rows")
norm = _normalize_max_rows(cli_val)
if norm is not None:
os.environ["EP_MAX_DATASET_ROWS"] = norm
num_runs_val = config.getoption("--ep-num-runs")
norm_runs = _normalize_number(num_runs_val)
if norm_runs is not None:
os.environ["EP_NUM_RUNS"] = norm_runs
max_concurrent_rollouts_val = config.getoption("--ep-max-concurrent-rollouts")
norm_concurrent_rollouts = _normalize_number(max_concurrent_rollouts_val)
if norm_concurrent_rollouts is not None:
os.environ["EP_MAX_CONCURRENT_ROLLOUTS"] = norm_concurrent_rollouts
max_concurrent_evals_val = config.getoption("--ep-max-concurrent-evaluations")
norm_concurrent_evals = _normalize_number(max_concurrent_evals_val)
if norm_concurrent_evals is not None:
os.environ["EP_MAX_CONCURRENT_EVALUATIONS"] = norm_concurrent_evals
if config.getoption("--ep-print-summary"):
os.environ["EP_PRINT_SUMMARY"] = "1"
summary_json_path = config.getoption("--ep-summary-json")
if summary_json_path:
os.environ["EP_SUMMARY_JSON"] = summary_json_path
max_retry = config.getoption("--ep-max-retry")
norm_max_retry = _normalize_number(max_retry)
if norm_max_retry is not None:
os.environ["EP_MAX_RETRY"] = norm_max_retry
fail_on_max_retry = config.getoption("--ep-fail-on-max-retry")
if fail_on_max_retry is not None:
os.environ["EP_FAIL_ON_MAX_RETRY"] = fail_on_max_retry
success_threshold_val = config.getoption("--ep-success-threshold")
se_threshold_val = config.getoption("--ep-se-threshold")
norm_success = _normalize_success_threshold(success_threshold_val)
norm_se = _normalize_se_threshold(se_threshold_val)
threshold_env = _build_passed_threshold_env(norm_success, norm_se)
if threshold_env is not None:
os.environ["EP_PASSED_THRESHOLD"] = threshold_env
if config.getoption("--ep-output-dir"):
# set this to save eval results to the target dir in jsonl format
os.environ["EP_OUTPUT_DIR"] = config.getoption("--ep-output-dir")
if config.getoption("--ep-no-op-rollout-processor"):
os.environ["EP_USE_NO_OP_ROLLOUT_PROCESSOR"] = "1"
if config.getoption("--ep-no-upload"):
os.environ["EP_NO_UPLOAD"] = "1"
if config.getoption("--ep-jsonl-path"):
os.environ["EP_JSONL_PATH"] = config.getoption("--ep-jsonl-path")
if config.getoption("--ep-completion-params"):
# redump to json to make sure they are legit
os.environ["EP_COMPLETION_PARAMS"] = json.dumps(
[json.loads(s) for s in config.getoption("--ep-completion-params") or []]
)
# Allow ad-hoc overrides of input params via CLI flags
try:
merged: dict = {}
input_params_opts = config.getoption("--ep-input-param")
if input_params_opts:
for opt in input_params_opts:
if opt is None:
continue
opt = str(opt)
if opt.startswith("@"): # load JSON file
p = pathlib.Path(opt[1:])
if p.is_file():
with open(p, "r", encoding="utf-8") as f:
obj = json.load(f)
if isinstance(obj, dict):
merged.update(obj)
elif "=" in opt:
k, v = opt.split("=", 1)
# Try parse JSON values, fallback to string
try:
merged[k] = json.loads(v)
except Exception:
merged[k] = v
reasoning_effort = config.getoption("--ep-reasoning-effort")
if reasoning_effort:
# Always place under extra_body to avoid LiteLLM rejecting top-level params
eb = merged.setdefault("extra_body", {})
# Convert "none" string to None value for API compatibility
eb["reasoning_effort"] = None if reasoning_effort.lower() == "none" else str(reasoning_effort)
if merged:
os.environ["EP_INPUT_PARAMS_JSON"] = json.dumps(merged)
except Exception:
# best effort, do not crash pytest session
pass
def _print_experiment_links(session):
"""Print all collected Fireworks experiment links from pytest stash."""
try:
# Late import to avoid circulars; if missing key, skip printing
EXPERIMENT_LINKS_STASH_KEY = None
try:
from .store_experiment_link import EXPERIMENT_LINKS_STASH_KEY as _KEY # type: ignore
EXPERIMENT_LINKS_STASH_KEY = _KEY
except Exception:
EXPERIMENT_LINKS_STASH_KEY = None
# Get links from pytest stash
links = []
if EXPERIMENT_LINKS_STASH_KEY is not None and EXPERIMENT_LINKS_STASH_KEY in session.stash:
links = session.stash[EXPERIMENT_LINKS_STASH_KEY]
# Only print when there is at least one successful link.
# Suppress the entire section if all links are failures (noise).
if any(link.get("status") == "success" for link in links):
print("\n" + "=" * 80, file=sys.__stderr__)
print("🔥 FIREWORKS EXPERIMENT LINKS", file=sys.__stderr__)
print("=" * 80, file=sys.__stderr__)
for link in links:
print(f"Experiment {link['experiment_id']}: {link['job_link']}", file=sys.__stderr__)
print("=" * 80, file=sys.__stderr__)
return True
return False
except Exception:
return False
def _print_local_ui_results_urls(session):
"""Print all collected evaluation results URLs from pytest stash."""
try:
# Late import to avoid circulars; if missing key, skip printing
RESULTS_URLS_STASH_KEY = None
try:
from .store_results_url import RESULTS_URLS_STASH_KEY as _URL_KEY # type: ignore
RESULTS_URLS_STASH_KEY = _URL_KEY
except Exception:
RESULTS_URLS_STASH_KEY = None
# Get URLs from pytest stash
urls_dict = {}
if RESULTS_URLS_STASH_KEY is not None and RESULTS_URLS_STASH_KEY in session.stash:
urls_dict = session.stash[RESULTS_URLS_STASH_KEY]
if urls_dict:
print("\n" + "=" * 80, file=sys.__stderr__)
print("📊 LOCAL UI EVALUATION RESULTS", file=sys.__stderr__)
print("=" * 80, file=sys.__stderr__)
for url_data in urls_dict.values():
print(f"📊 Invocation {url_data['invocation_id']}:", file=sys.__stderr__)
print(f" 📊 Aggregate scores: {url_data['pivot_url']}", file=sys.__stderr__)
print(f" 📋 Trajectories: {url_data['table_url']}", file=sys.__stderr__)
print("=" * 80, file=sys.__stderr__)
return True
return False
except Exception:
return False
def pytest_sessionfinish(session, exitstatus):
"""Print all collected Fireworks experiment links and evaluation results URLs from pytest stash."""
try:
# Print experiment links and results URLs separately
links_printed = _print_experiment_links(session)
urls_printed = _print_local_ui_results_urls(session)
# Flush stderr if anything was printed
if links_printed or urls_printed:
err_stream = getattr(sys, "__stderr__", None)
if err_stream is not None:
try:
err_stream.flush() # type: ignore[attr-defined]
except Exception:
pass
except Exception:
pass