forked from eval-protocol/python-sdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathlangfuse_example.py
More file actions
209 lines (171 loc) · 7.46 KB
/
Copy pathlangfuse_example.py
File metadata and controls
209 lines (171 loc) · 7.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
"""
Langfuse Adapter Example
This example demonstrates how to use the Langfuse adapter to pull data from
a Langfuse deployment and convert it to EvaluationRow format for evaluation.
"""
import os
from datetime import datetime, timedelta
from typing import List
from eval_protocol.adapters.langfuse import create_langfuse_adapter
from eval_protocol.models import EvaluationRow
def main():
"""Example usage of the Langfuse adapter."""
# Configuration - you can set these as environment variables
public_key = os.getenv("LANGFUSE_PUBLIC_KEY", "your_public_key_here")
secret_key = os.getenv("LANGFUSE_SECRET_KEY", "your_secret_key_here")
host = os.getenv("LANGFUSE_HOST", "https://langfuse-web-prod-zfdbl7ykrq-uc.a.run.app")
project_id = os.getenv("LANGFUSE_PROJECT_ID", "cmdj5yxhk0006s6022cyi0prv")
print(f"Connecting to Langfuse at: {host}")
print(f"Project ID: {project_id}\n")
# Create the adapter
try:
adapter = create_langfuse_adapter(
public_key=public_key,
secret_key=secret_key,
host=host,
project_id=project_id,
)
print("✅ Langfuse adapter created successfully")
except ImportError as e:
print(f"❌ Error: {e}")
print("Install Langfuse dependencies: pip install 'eval-protocol[langfuse]'")
return
except Exception as e:
print(f"❌ Failed to create adapter: {e}")
return
# Example 1: Get recent evaluation rows
print("\n📊 Example 1: Get recent evaluation rows")
try:
rows = list(
adapter.get_evaluation_rows(
limit=5,
from_timestamp=datetime.now() - timedelta(days=7),
include_tool_calls=True,
)
)
print(f"Retrieved {len(rows)} evaluation rows")
for i, row in enumerate(rows):
print(f" Row {i + 1}:")
print(f" - ID: {row.input_metadata.row_id if row.input_metadata else 'N/A'}")
print(f" - Messages: {len(row.messages)}")
print(f" - Has tools: {'Yes' if row.tools else 'No'}")
print(f" - Ground truth: {'Yes' if row.ground_truth else 'No'}")
# Show first message content (truncated)
if row.messages:
content = row.messages[0].content or ""
preview = content[:100] + "..." if len(content) > 100 else content
print(f" - First message: {preview}")
print()
except Exception as e:
print(f"❌ Error retrieving rows: {e}")
# Example 2: Filter by specific criteria
print("\n🔍 Example 2: Filter by specific criteria")
try:
rows = list(
adapter.get_evaluation_rows(
limit=3,
tags=["production"], # Filter by tags if available
include_tool_calls=True,
)
)
print(f"Retrieved {len(rows)} rows with 'production' tag")
except Exception as e:
print(f"❌ Error with filtered query: {e}")
# Example 3: Get specific traces by ID
print("\n🎯 Example 3: Get specific traces by ID")
try:
# Replace with actual trace IDs from your Langfuse deployment
trace_ids = ["trace_id_1", "trace_id_2"] # These would be real IDs
rows = list(
adapter.get_evaluation_rows_by_ids(
trace_ids=trace_ids,
include_tool_calls=True,
)
)
print(f"Retrieved {len(rows)} rows by specific IDs")
except Exception as e:
print(f"❌ Error retrieving specific traces: {e}")
# Example 4: Extract different types of conversations
print("\n💬 Example 4: Analyze conversation types")
try:
rows = list(adapter.get_evaluation_rows(limit=10, include_tool_calls=True))
chat_only = []
tool_calling = []
for row in rows:
if row.tools and any(
msg.tool_calls for msg in row.messages if hasattr(msg, "tool_calls") and msg.tool_calls
):
tool_calling.append(row)
else:
chat_only.append(row)
print(f"Chat-only conversations: {len(chat_only)}")
print(f"Tool calling conversations: {len(tool_calling)}")
# Show example of tool calling conversation
if tool_calling:
row = tool_calling[0]
print("\n🔧 Example tool calling conversation:")
for i, msg in enumerate(row.messages):
print(f" {i + 1}. {msg.role}: {msg.content[:50] if msg.content else '[No content]'}...")
if hasattr(msg, "tool_calls") and msg.tool_calls:
for tool_call in msg.tool_calls:
print(f" 🛠 Tool call: {tool_call}")
except Exception as e:
print(f"❌ Error analyzing conversation types: {e}")
def demonstrate_evaluation_integration():
"""Show how to use Langfuse data with evaluation functions."""
print("\n🧪 Integration with Evaluation Functions")
# This would typically be in a separate evaluation script
try:
from eval_protocol.rewards.math import math_reward
# Create adapter (reuse configuration from main example)
adapter = create_langfuse_adapter(
public_key=os.getenv("LANGFUSE_PUBLIC_KEY", "your_public_key_here"),
secret_key=os.getenv("LANGFUSE_SECRET_KEY", "your_secret_key_here"),
host=os.getenv("LANGFUSE_HOST", "https://langfuse-web-prod-zfdbl7ykrq-uc.a.run.app"),
project_id=os.getenv("LANGFUSE_PROJECT_ID", "cmdj5yxhk0006s6022cyi0prv"),
)
# Get data and evaluate
rows = list(adapter.get_evaluation_rows(limit=3))
for i, row in enumerate(rows):
print(f"\nEvaluating row {i + 1}:")
# Only evaluate if we have ground truth
if row.ground_truth:
try:
result = math_reward(
messages=row.messages,
ground_truth=row.ground_truth,
)
print(f" Math evaluation score: {result.score:.2f}")
print(f" Reason: {result.reason}")
except Exception as e:
print(f" ❌ Evaluation failed: {e}")
else:
print(" ⚠️ No ground truth available for evaluation")
except ImportError:
print("Math reward function not available")
except Exception as e:
print(f"❌ Error in evaluation integration: {e}")
if __name__ == "__main__":
print("🚀 Langfuse Adapter Example")
print("=" * 50)
# Check if credentials are set
if not all(
[
os.getenv("LANGFUSE_PUBLIC_KEY"),
os.getenv("LANGFUSE_SECRET_KEY"),
]
):
print("⚠️ To run this example with real data, set environment variables:")
print(" export LANGFUSE_PUBLIC_KEY='your_public_key'")
print(" export LANGFUSE_SECRET_KEY='your_secret_key'")
print(" export LANGFUSE_HOST='your_langfuse_host' # optional")
print(" export LANGFUSE_PROJECT_ID='your_project_id' # optional")
print()
main()
demonstrate_evaluation_integration()
print("\n✅ Example completed!")
print("\nNext steps:")
print("1. Set up your Langfuse credentials")
print("2. Modify the filters and parameters to match your data")
print("3. Integrate with your evaluation pipeline")
print("4. Use the converted EvaluationRow data for training or evaluation")