forked from eval-protocol/python-sdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathquickstart.py
More file actions
57 lines (49 loc) · 2.37 KB
/
Copy pathquickstart.py
File metadata and controls
57 lines (49 loc) · 2.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# REMOTE SERVER OPTIONS:
#
# Option 1: Use Vercel dev server locally (recommended for development)
# cd eval_protocol/quickstart/svg_agent/vercel_svg_server
# vercel dev
# Then change remote_base_url to: "http://localhost:3000"
#
# Option 2: Use deployed Vercel production server (current configuration)
# No setup needed - uses the deployed serverless function
# Currently using: https://vercel-svg-server-qntltzfaq-xzrdereks-projects.vercel.app
#
# Option 3: Use local Python server (for testing)
# python -m tests.remote_server.remote_server
# Then change remote_base_url to: "http://127.0.0.1:3000"
import os
from typing import List
import pytest
from eval_protocol.data_loader.dynamic_data_loader import DynamicDataLoader
from eval_protocol.models import EvaluationRow, Message
from eval_protocol.pytest import evaluation_test
from eval_protocol.pytest.remote_rollout_processor import RemoteRolloutProcessor
def rows() -> List[EvaluationRow]:
row = EvaluationRow(messages=[Message(role="user", content="What is the capital of France?")])
return [row, row, row]
@pytest.mark.skipif(os.environ.get("CI") == "true", reason="Only run this test locally (skipped in CI)")
@pytest.mark.parametrize("completion_params", [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b"}])
@evaluation_test(
data_loaders=DynamicDataLoader(
generators=[rows],
),
rollout_processor=RemoteRolloutProcessor(
# For local Vercel dev: "http://localhost:3000"
# For production Vercel: (current setting)
remote_base_url="https://vercel-svg-server.vercel.app",
timeout_seconds=30,
),
)
async def test_remote_rollout_and_fetch_fireworks(row: EvaluationRow) -> EvaluationRow:
"""
End-to-end test with Vercel production server:
- Uses deployed Vercel serverless function (no manual startup needed)
- trigger remote rollout via RemoteRolloutProcessor (calls init/status)
- fetch traces from Fireworks tracing (uses default FireworksTracingAdapter)
- FAIL if no traces found or rollout_id missing
"""
assert row.messages[0].content == "What is the capital of France?", "Row should have correct message content"
assert len(row.messages) > 1, "Row should have a response. If this fails, we fellback to the original row."
assert row.execution_metadata.rollout_id, "Row should have a rollout_id from the remote rollout"
return row