Repository navigation
250 lines (233 loc) · 10.8 KB
/
Copy patheval-coder-agents.yaml
File metadata and controls
250 lines (233 loc) · 10.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
# A/B Coder Agents on a PR branch vs main: each arm builds its own coderd,
# runs Terminal-Bench 2.0 through Harbor with the Coder environment provider
# (scripts/harbor-coder), and the comparison is posted on the PR.
#
# Trigger by commenting `/coder-agents-eval` on a PR (push access required,
# fork PRs refused since the candidate arm runs the PR's code with the gateway
# token in scope), or via workflow_dispatch to compare the current ref to main.
# Trials spend real model tokens, so this never runs automatically.
name: eval/coder-agents
on:
issue_comment:
types: [created]
workflow_dispatch:
inputs:
pr:
description: PR number to post the comparison on (optional)
required: false
type: string
permissions: {}
# Non-eval events (every other PR comment) get a group of their own so they
# cannot replace a queued /coder-agents-eval run, which would cancel it.
concurrency:
group: >-
${{ (github.event_name == 'workflow_dispatch' || startsWith(github.event.comment.body, '/coder-agents-eval'))
&& format('eval-coder-agents-{0}', github.event.issue.number || github.ref)
|| format('eval-coder-agents-other-{0}', github.run_id) }}
cancel-in-progress: false
env:
HARBOR_DATASET: terminal-bench@2.0
# qemu-* and build-cython-ext are broken upstream (bit-rotted apt sources
# and a git clone in the verifier). The rest are ML training, emulation,
# and long compiles that each take 20 to 60 minutes and are not decided by
# the system prompt; dropping them cuts an arm from ~3h to ~80 minutes.
# crack-7z-hash, break-filter-js-from-html, and vulnerable-secret are
# always refused by Anthropic's content policy (cyber), so they score 0 in
# every arm and cannot move the comparison.
HARBOR_EXCLUDE: >-
-x qemu-* -x build-cython-ext
-x train-fasttext -x compile-compcert -x install-windows-3.11
-x winning-avg-corewars -x extract-moves-from-video -x mcmc-sampling-stan
-x regex-chess -x mteb-leaderboard -x caffe-cifar-10
-x crack-7z-hash -x break-filter-js-from-html -x vulnerable-secret
# Sonnet sits mid-range on Terminal-Bench, leaving headroom in both
# directions for a prompt change to move; Opus is near the ceiling.
HARBOR_CODER_MODEL: claude-sonnet-5-5
HARBOR_ATTEMPTS: "2"
# chatd admits at most 5 concurrent root chats without a license
# (coderd/x/chatd/agentadmission.go). Above that, trials queue on Harbor's
# timeout and score 0 depending on start order.
HARBOR_CONCURRENCY: "5"
jobs:
resolve:
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event.issue.pull_request && startsWith(github.event.comment.body, '/coder-agents-eval'))
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
pull-requests: write
outputs:
candidate_ref: ${{ steps.pr.outputs.sha || github.sha }}
candidate_label: ${{ steps.pr.outputs.ref || github.ref_name }}
baseline_ref: ${{ steps.base.outputs.sha }}
pr_number: ${{ github.event.issue.number || inputs.pr }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
with:
egress-policy: audit
- name: Resolve PR
id: pr
if: github.event_name == 'issue_comment'
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR: ${{ github.event.issue.number }}
COMMENTER: ${{ github.event.comment.user.login }}
COMMENT_ID: ${{ github.event.comment.id }}
run: |
set -euo pipefail
if [[ "$(gh api "/repos/${REPO}/collaborators/${COMMENTER}/permission" --jq .user.permissions.push)" != "true" ]]; then
echo "::error::${COMMENTER} does not have push access"; exit 1
fi
if [[ "$(gh pr view "${PR}" -R "${REPO}" --json isCrossRepository --jq .isCrossRepository)" == "true" ]]; then
echo "::error::refusing to run on a fork PR"; exit 1
fi
echo "sha=$(gh pr view "${PR}" -R "${REPO}" --json headRefOid --jq .headRefOid)" >> "${GITHUB_OUTPUT}"
echo "ref=$(gh pr view "${PR}" -R "${REPO}" --json headRefName --jq .headRefName)" >> "${GITHUB_OUTPUT}"
gh api -X POST "/repos/${REPO}/issues/comments/${COMMENT_ID}/reactions" -f content=eyes >/dev/null ||
echo "::warning::could not react to comment ${COMMENT_ID}"
# Compare against the commit the candidate branched from, so commits
# that landed on main since then are not attributed to the candidate.
- name: Resolve baseline
id: base
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
CANDIDATE: ${{ steps.pr.outputs.sha || github.sha }}
run: |
set -euo pipefail
sha="$(gh api "/repos/${REPO}/compare/main...${CANDIDATE}" --jq .merge_base_commit.sha)"
echo "baseline: ${sha}, candidate: ${CANDIDATE}"
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
run:
name: run (${{ matrix.arm }})
needs: resolve
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
environment: main
timeout-minutes: 360
permissions:
contents: read
strategy:
fail-fast: false
matrix:
arm: [baseline, candidate]
steps:
- name: Harden Runner
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
with:
egress-policy: audit
- name: Checkout arm ref
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ matrix.arm == 'baseline' && needs.resolve.outputs.baseline_ref || needs.resolve.outputs.candidate_ref }}
fetch-depth: 1
persist-credentials: false
# The harness always comes from the candidate so both arms run the same code.
- name: Checkout harness from candidate ref
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve.outputs.candidate_ref }}
path: .eval-src
sparse-checkout: scripts/harbor-coder
persist-credentials: false
- run: rm -rf scripts/harbor-coder && cp -r .eval-src/scripts/harbor-coder scripts/harbor-coder
- uses: ./.github/actions/setup-go-paths
- uses: ./.github/actions/setup-mise
with:
install-args: "go"
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
- name: Install Harbor from the hash-pinned lock
run: |
./scripts/harbor-coder/install.sh "${RUNNER_TEMP}/harbor-venv"
echo "${RUNNER_TEMP}/harbor-venv/bin" >> "${GITHUB_PATH}"
- name: Build and serve coderd
env:
# A Coder API key for dogfood; the eval's coderd uses dogfood's AI
# gateway as its model upstream, so no provider key is involved.
HARBOR_CODER_UPSTREAM_TOKEN: ${{ secrets.EVAL_CODER_SESSION_TOKEN }}
run: ./scripts/harbor-coder/serve-coderd.sh "${RUNNER_TEMP}/coder.env"
- name: Run Harbor
env:
ARM: ${{ matrix.arm }}
run: |
set -euo pipefail
set -a; . "${RUNNER_TEMP}/coder.env"; set +a
export PATH="${HARBOR_CODER_BIN%/*}:${PATH}"
# shellcheck disable=SC2086
harbor run -d "${HARBOR_DATASET}" ${HARBOR_EXCLUDE} \
--agent harbor_coder.agent:CoderAgentsAgent \
--env harbor_coder.environment:CoderEnvironment \
-k "${HARBOR_ATTEMPTS}" -n "${HARBOR_CONCURRENCY}" -o jobs --job-name "${ARM}"
# Harbor exits 0 even when every trial raised.
if [[ "$(find "jobs/${ARM}" -name reward.txt -o -name reward.json | wc -l)" -eq 0 ]]; then
echo "::error::no trial produced a reward"; exit 1
fi
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: harbor-${{ matrix.arm }}
path: jobs/
# Only the log: the rest of .harbor-coder holds the owner password and
# session token. A separate artifact keeps the jobs/ layout compare expects.
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: coderd-log-${{ matrix.arm }}
path: .harbor-coder/coderd.log
include-hidden-files: true
if-no-files-found: ignore
compare:
needs: [resolve, run]
if: always() && needs.resolve.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve.outputs.candidate_ref }}
sparse-checkout: scripts/harbor-coder
persist-credentials: false
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
continue-on-error: true
with:
pattern: harbor-*
path: jobs
merge-multiple: true
- name: Compare
env:
LABEL: ${{ needs.resolve.outputs.candidate_label }}
RUN_RESULT: ${{ needs.run.result }}
BASELINE_SHA: ${{ needs.resolve.outputs.baseline_ref }}
CANDIDATE_SHA: ${{ needs.resolve.outputs.candidate_ref }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
if [[ "${RUN_RESULT}" != "success" ]]; then
# A partial table would read as a verdict; the run log has the cause.
printf '## Coder Agents eval: %s vs main\n\n**The eval did not complete** (run: %s). See the [run](%s) for the failing arm.\n' \
"${LABEL}" "${RUN_RESULT}" "${RUN_URL}" > report.md
else
mkdir -p jobs/baseline jobs/candidate
PYTHONPATH=scripts/harbor-coder python3 -m harbor_coder.compare \
--baseline jobs/baseline --candidate jobs/candidate --labels main "${LABEL}" > report.md
printf '\n%s, %s, %s attempt(s) per task. Baseline %s (merge base with main), candidate %s. [Run](%s)\n' \
"${HARBOR_DATASET}" "${HARBOR_CODER_MODEL}" "${HARBOR_ATTEMPTS}" "${BASELINE_SHA:0:10}" "${CANDIDATE_SHA:0:10}" "${RUN_URL}" >> report.md
fi
cat report.md >> "${GITHUB_STEP_SUMMARY}"
- name: Comment on PR
if: needs.resolve.outputs.pr_number != ''
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR: ${{ needs.resolve.outputs.pr_number }}
run: gh pr comment "${PR}" -R "${REPO}" --body-file report.md