forked from eval-protocol/python-sdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmain.py
More file actions
42 lines (32 loc) · 1.48 KB
/
Copy pathmain.py
File metadata and controls
42 lines (32 loc) · 1.48 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
"""
APPS Coding Evaluation Example
This example shows how to create a custom reward function for code generation
that reuses existing reward-kit functionality. Here we import and use the
built-in APPS coding evaluation to test code correctness against test cases.
"""
from typing import Any, Dict, List, Optional, Union
from eval_protocol import EvaluateResult, reward_function
from eval_protocol.models import Message
# Import the existing reward function from reward-kit
from eval_protocol.rewards.apps_coding_reward import evaluate_apps_solution
@reward_function
def evaluate(
messages: Union[List[Message], List[Dict[str, Any]]],
ground_truth: Optional[str] = None,
**kwargs,
) -> EvaluateResult:
"""
Evaluate code generation accuracy using execution against test cases.
This function demonstrates how to reuse existing reward-kit functions
for code evaluation while allowing for future customization if needed.
Args:
messages: The conversation messages including the generated code
ground_truth: JSON string with test cases (inputs/outputs)
**kwargs: Additional parameters (like execution_timeout)
Returns:
EvaluateResult with score and metrics
"""
# For now, we directly use the built-in APPS evaluation function
# In a real scenario, you might add preprocessing, custom logic, or
# combine multiple reward functions here
return evaluate_apps_solution(messages=messages, ground_truth=ground_truth, **kwargs)