-
-
Notifications
You must be signed in to change notification settings - Fork 81
Expand file tree
/
Copy pathinitialGeneration.eval.ts
More file actions
93 lines (86 loc) · 2.68 KB
/
Copy pathinitialGeneration.eval.ts
File metadata and controls
93 lines (86 loc) · 2.68 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
import * as braintrust from 'braintrust';
import { SUGGESTIONS } from 'codinit-agent/constants.js';
import { mkdtempSync } from 'fs';
import path from 'path';
import os from 'os';
import { anthropic } from '@ai-sdk/anthropic';
import { openai } from '@ai-sdk/openai';
import { google } from '@ai-sdk/google';
import { xai } from '@ai-sdk/xai';
import { codinitTask } from './codinitTask.js';
import { codinitScorer } from './codinitScorer.js';
import type { CodinitModel } from './types.js';
import * as net from 'net';
const CHEF_PROJECT = 'codinit';
function codinitEval(model: CodinitModel) {
const experimentName = `${CHEF_PROJECT}-${model.name}`;
let outputDir = process.env.OUTPUT_TEMPDIR;
if (!outputDir) {
outputDir = mkdtempSync(path.join(os.tmpdir(), 'codinit-eval'));
}
const environment = process.env.ENVIRONMENT ?? 'dev';
return braintrust.Eval(CHEF_PROJECT, {
experimentName,
data: SUGGESTIONS.map((s) => ({ input: s.prompt })),
task: (input: string) => codinitTask(model, outputDir!, input),
scores: [codinitScorer],
maxConcurrency: 2,
metadata: {
model: model.name,
model_slug: model.model_slug,
environment,
tempdir: outputDir,
},
});
}
// This is tricky: Node v17 and higher resolve `localhost` IPv6 (::1), which can fail
// if the server only binds to IPv4. Use `setDefaultAutoSelectFamily(true)` to tell
// Node to use Happy Eyeballs to detect IPv6 support.
// Source: https://github.com/nuxt/nuxt/issues/12358
net.setDefaultAutoSelectFamily(true);
if (process.env.ANTHROPIC_API_KEY) {
codinitEval({
name: 'claude-4-sonnet',
model_slug: 'claude-sonnet-4-20250514',
ai: anthropic('claude-sonnet-4-20250514'),
maxTokens: 16384,
});
codinitEval({
name: 'claude-4.5-sonnet',
model_slug: 'claude-sonnet-4-5',
ai: anthropic('claude-sonnet-4-5'),
maxTokens: 16384,
});
}
// Braintrust sets the OPENAI_API_KEY environment variable even if we don't set it, so we need
// to manually check the USE_OPENAI environment variable to determine if we should use OpenAI.
if (process.env.OPENAI_API_KEY && process.env.USE_OPENAI === 'true') {
codinitEval({
name: 'gpt-4.1',
model_slug: 'gpt-4.1',
ai: openai('gpt-4.1'),
maxTokens: 8192,
});
codinitEval({
name: 'gpt-5',
model_slug: 'gpt-5',
ai: openai('gpt-5'),
maxTokens: 8192,
});
}
if (process.env.GOOGLE_GENERATIVE_AI_API_KEY) {
codinitEval({
name: 'gemini-2.5-pro',
model_slug: 'gemini-2.5-pro',
ai: google('gemini-2.5-pro'),
maxTokens: 20000,
});
}
if (process.env.XAI_API_KEY) {
codinitEval({
name: 'grok-3-mini',
model_slug: 'grok-3-mini',
ai: xai('grok-3-mini'),
maxTokens: 8192,
});
}