forked from eval-protocol/python-sdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmain.py
More file actions
88 lines (71 loc) · 3.2 KB
/
Copy pathmain.py
File metadata and controls
88 lines (71 loc) · 3.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""
Math Evaluation Example
This example shows how to create a custom reward function for math problems
that combines accuracy checking with format validation. The math example
expects answers to be in <think>...</think><answer>...</answer> format.
"""
import re
from typing import Any, Dict, List, Optional, Union
from eval_protocol import EvaluateResult, MetricResult, reward_function
from eval_protocol.models import Message
# Import the existing reward function from reward-kit
from eval_protocol.rewards.math import math_reward
def check_think_answer_format(text: str) -> bool:
"""Check if text follows <think>...</think><answer>...</answer> format."""
if not text:
return False
pattern = r"^<think>[\s\S]*?</think>\s*<answer>[\s\S]*?</answer>$"
return bool(re.match(pattern, text.strip()))
@reward_function
def evaluate(
messages: Union[List[Message], List[Dict[str, Any]]],
ground_truth: Optional[str] = None,
**kwargs,
) -> EvaluateResult:
"""
Evaluate math problem solving considering both accuracy and format.
This function demonstrates how to combine multiple evaluation criteria:
- Numerical accuracy using built-in math evaluation
- Format compliance checking for <think>...</think><answer>...</answer> structure
Args:
messages: The conversation messages including the math solution
ground_truth: Expected answer for comparison
**kwargs: Additional parameters (like tolerance)
Returns:
EvaluateResult with combined score and detailed metrics
"""
# Get the assistant's response
assistant_message = messages[-1]
if isinstance(assistant_message, dict):
assistant_response = assistant_message.get("content", "")
else:
assistant_response = assistant_message.content or ""
# Evaluate numerical accuracy using built-in function
accuracy_result = math_reward(messages=messages, ground_truth=ground_truth, **kwargs)
# Evaluate format compliance (looking for <think>...</think><answer>...</answer> format)
format_correct = check_think_answer_format(assistant_response)
format_score = 1.0 if format_correct else 0.0
# The combined score is a weighted average of accuracy and format
weights = {"accuracy": 0.8, "format": 0.2}
combined_score = (accuracy_result.score * weights["accuracy"]) + (format_score * weights["format"])
# If accuracy is 0, the overall score is 0, regardless of format.
if accuracy_result.score == 0.0:
combined_score = 0.0
# Create metrics structure expected by tests
metrics = {
"accuracy_reward": MetricResult(
score=accuracy_result.score,
reason=f"Numerical accuracy: {accuracy_result.reason}",
is_score_valid=True,
),
"format_reward": MetricResult(
score=format_score,
reason=f"Format compliance: {'correct' if format_correct else 'incorrect'} <think>...</think><answer>...</answer> structure",
is_score_valid=True,
),
}
return EvaluateResult(
score=combined_score,
reason=f"Combined score: {combined_score:.2f} (accuracy: {accuracy_result.score:.2f}, format: {format_score:.2f})",
metrics=metrics,
)