forked from eval-protocol/python-sdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_server.py
More file actions
99 lines (81 loc) · 3.54 KB
/
Copy pathtest_server.py
File metadata and controls
99 lines (81 loc) · 3.54 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
from typing import Any, Dict, List, Optional
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from eval_protocol.models import EvaluateResult, MetricResult
from eval_protocol.server import create_app
@pytest.fixture
def test_reward_func():
"""Fixture that returns a test reward function."""
def _reward_func(
messages: List[Dict[str, str]],
original_messages: Optional[List[Dict[str, str]]] = None,
**kwargs,
) -> EvaluateResult:
"""Test reward function that returns a simple score."""
metrics = {"test": MetricResult(score=0.5, success=True, reason="Test reason")}
return EvaluateResult(score=0.5, reason="Test score reason", metrics=metrics)
return _reward_func
class TestServer:
"""Tests for the FastAPI server."""
@pytest.fixture
def client(self, test_reward_func):
"""Create a test client for the FastAPI app."""
app = create_app(test_reward_func)
return TestClient(app)
def test_health_endpoint(self, client):
"""Test the health check endpoint."""
response = client.get("/health")
assert response.status_code == 200
assert response.json() == {"status": "ok"}
def test_reward_endpoint(self, client):
"""Test the reward endpoint."""
payload: Dict[str, Any] = {
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there"},
],
"original_messages": [{"role": "user", "content": "Hello"}],
}
response = client.post("/reward", json=payload)
assert response.status_code == 200
data = response.json()
assert data["score"] == 0.5
assert data["reason"] == "Test score reason"
assert "metrics" in data
assert "test" in data["metrics"]
assert data["metrics"]["test"]["score"] == 0.5
assert data["metrics"]["test"]["reason"] == "Test reason"
assert data["metrics"]["test"]["is_score_valid"] is True
def test_reward_endpoint_with_metadata(self, client):
"""Test the reward endpoint with metadata."""
payload: Dict[str, Any] = {
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there"},
],
"original_messages": [{"role": "user", "content": "Hello"}],
"metadata": {"test_key": "test_value"},
}
response = client.post("/reward", json=payload)
assert response.status_code == 200
data = response.json()
assert data["score"] == 0.5
def test_reward_endpoint_missing_required_fields(self, client):
"""Test the reward endpoint with missing required fields."""
# Empty payload without messages field
payload: Dict[str, Any] = {}
response = client.post("/reward", json=payload)
assert response.status_code == 422 # Validation error
def test_reward_endpoint_malformed_messages(self, client):
"""Test the reward endpoint with malformed messages."""
# Malformed messages - missing role
payload: Dict[str, Any] = {
"messages": [
{"content": "Hello"}, # Missing role
{"role": "assistant", "content": "Hi there"},
],
"original_messages": [{"role": "user", "content": "Hello"}],
}
response = client.post("/reward", json=payload)
assert response.status_code == 422 # Validation error