forked from eval-protocol/python-sdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathn_variant_example.yaml
More file actions
31 lines (25 loc) · 845 Bytes
/
Copy pathn_variant_example.yaml
File metadata and controls
31 lines (25 loc) · 845 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# Example configuration for N-variant generation
# This configuration will generate 5 different responses for each sample
generation:
enabled: true
model_name: "accounts/fireworks/models/llama-v3p1-8b-instruct"
n: 5 # Generate 5 variants for each sample
cache:
enabled: true
api_params:
max_concurrent_requests: 10
temperature: 0.8 # Higher temperature for more diverse responses
top_p: 0.9
reward:
function_path: "eval_protocol.rewards.accuracy.accuracy_reward"
params: {}
dataset:
_target_: eval_protocol.datasets.loader.load_and_process_dataset
path_or_name: "path/to/your/dataset.jsonl"
output:
results_file: "n_variant_results.jsonl"
preview_pairs_file: "n_variant_preview.jsonl"
evaluation_params:
limit_samples: 10 # Limit to 10 samples for testing
logging_params:
batch_log_interval: 5