Repository navigation
378 lines (350 loc) · 16.6 KB
/
Copy pathsync_typeshed.yaml
File metadata and controls
378 lines (350 loc) · 16.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
name: Sync typeshed
# How this works:
#
# 1. A Linux worker:
# a. Checks out Ruff and typeshed
# b. Deletes the vendored typeshed stdlib stubs from Ruff
# c. Copies the latest versions of the stubs from typeshed
# d. Uses docstring-adder to sync all docstrings available on Linux
# e. Creates a new branch on the upstream astral-sh/ruff repository
# f. Commits the changes it's made and pushes them to the new upstream branch
# 2. Once the Linux worker is done, a Windows worker:
# a. Checks out the branch created by the Linux worker
# b. Syncs all docstrings available on Windows that are not available on Linux
# c. Commits the changes and pushes them to the same upstream branch
# 3. Once the Windows worker is done, a MacOS worker:
# a. Checks out the branch created by the Linux worker
# b. Syncs all docstrings available on MacOS that are not available on Linux or Windows
# c. Formats the code again and pushes the completed sync
# d. Applies the typeshed patches in a standalone commit and pushes it
# 4. A second Linux worker attempts to update snapshots and uploads the changes as an artifact
# (this step is allowed to fail)
# 5. Once the snapshot job finishes, fails, or times out, a third Linux worker:
# a. Commits and pushes any snapshot changes uploaded in step 4
# b. Creates a PR against the `main` branch as long as step 3 succeeded
# 6. If any required step failed, an issue is created in the `astral-sh/ruff` repository
on:
workflow_dispatch:
schedule:
# Run on the 1st and the 15th of every month:
- cron: "0 0 1,15 * *"
defaults:
run:
shell: bash
permissions: {}
# Keep this workflow isolated from shared caches for security.
# All jobs use GitHub-hosted runners so GitHub enforces this restriction.
cache-mode: none
env:
UV_LOCKED: 1
# Don't set this flag globally for the workflow: it does strange things
# to the snapshots in the `cargo insta test --accept` step in the snapshot job.
#
# FORCE_COLOR: 1
CARGO_INCREMENTAL: 0
CARGO_NET_RETRY: 10
CARGO_PROFILE_DEV_DEBUG: line-tables-only
CARGO_PROFILE_DEV_LTO: "false"
CARGO_PROFILE_DEV_OPT_LEVEL: 0
CARGO_TERM_COLOR: always
NEXTEST_PROFILE: "ci"
RUSTUP_MAX_RETRIES: 10
GH_TOKEN: ${{ github.token }}
# The name of the upstream branch that the first worker creates,
# and which the later jobs update.
UPSTREAM_BRANCH: typeshedbot/sync-typeshed
# The path to the directory that contains the vendored typeshed stubs,
# relative to the root of the Ruff repository.
VENDORED_TYPESHED: crates/ty_vendored/vendor/typeshed
jobs:
# Sync typeshed stubs, and sync all docstrings available on Linux.
# Push the changes to a new branch on the upstream repository.
sync:
name: Sync typeshed
runs-on: ${{ github.repository == 'astral-sh/ruff' && 'github-ubuntu-24.04-x86_64-4' || 'ubuntu-latest' }}
timeout-minutes: 20
# Don't run the cron job on forks:
if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }}
permissions:
contents: write # to push back to the repository
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
name: Checkout Ruff
with:
path: ruff
persist-credentials: true
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
name: Checkout typeshed
with:
repository: python/typeshed
path: typeshed
persist-credentials: false
- name: Setup git
run: |
git config --global user.name typeshedbot
git config --global user.email '<>'
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
enable-cache: false
# renovate: datasource=github-release-attachments depName=astral-sh/uv
version: "0.13.1"
checksum: "eadd9d27c134ab5ac1ffd4473b97a741407f3ea238288f03121fc4deddf90220"
- name: Sync typeshed stubs
run: |
rm -rf "ruff/${VENDORED_TYPESHED}"
mkdir "ruff/${VENDORED_TYPESHED}"
cp typeshed/README.md "ruff/${VENDORED_TYPESHED}"
cp typeshed/LICENSE "ruff/${VENDORED_TYPESHED}"
# The pyproject.toml file is needed by a later job for the black configuration.
# It's deleted before creating the PR.
cp typeshed/pyproject.toml "ruff/${VENDORED_TYPESHED}"
cp -r typeshed/stdlib "ruff/${VENDORED_TYPESHED}/stdlib"
rm -rf "ruff/${VENDORED_TYPESHED}/stdlib/@tests"
git -C typeshed rev-parse HEAD > "ruff/${VENDORED_TYPESHED}/source_commit.txt"
cd ruff
git checkout -b "${UPSTREAM_BRANCH}"
git add .
git commit -m "Sync typeshed. Source commit: https://github.com/python/typeshed/commit/$(git -C ../typeshed rev-parse HEAD)" --allow-empty
- name: Sync Linux docstrings
if: ${{ success() }}
env:
FORCE_COLOR: 1
run: |
cd ruff
./scripts/codemod_docstrings.sh
git commit -am "Sync Linux docstrings" --allow-empty
- name: Push the changes
id: commit
if: ${{ success() }}
run: git -C ruff push --force --set-upstream origin "${UPSTREAM_BRANCH}"
# Checkout the branch created by the sync job,
# and sync all docstrings available on Windows that are not available on Linux.
# Commit the changes and push them to the same branch.
docstrings-windows:
runs-on: windows-latest
timeout-minutes: 20
needs: [sync]
# Don't run the cron job on forks.
# The job will also be skipped if the sync job failed, because it's specified in `needs` above,
# and we haven't used `always()` in the `if` condition here
# (https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-requiring-successful-dependent-jobs)
if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }}
permissions:
contents: write # to push back to the repository
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
name: Checkout Ruff
with:
persist-credentials: true
ref: ${{ env.UPSTREAM_BRANCH}}
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
enable-cache: false
# renovate: datasource=github-release-attachments depName=astral-sh/uv
version: "0.13.1"
checksum: "3b2661a463a5baea7f5076ea2b25ab71d77f36a6c0d809e5350651792f80a672"
- name: Setup git
run: |
git config --global user.name typeshedbot
git config --global user.email '<>'
- name: Sync Windows docstrings
id: docstrings
env:
FORCE_COLOR: 1
run: ./scripts/codemod_docstrings.sh
- name: Commit the changes
if: ${{ steps.docstrings.outcome == 'success' }}
run: |
git commit -am "Sync Windows docstrings" --allow-empty
git push
# - Checkout the branch created by the sync job.
# - Sync all docstrings available on macOS that are not available on Linux or Windows.
# - Format the stubs.
# - Apply the typeshed patches.
# - Push the changes to the same branch.
docstrings-macos:
runs-on: macos-latest
timeout-minutes: 20
needs: [sync, docstrings-windows]
# Don't run the cron job on forks.
# The job will also be skipped if the sync or docstrings-windows jobs failed,
# because they're specified in `needs` above and we haven't used an `always()` condition in the `if` here
# (https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-requiring-successful-dependent-jobs)
if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }}
permissions:
contents: write # to push back to the repository
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
name: Checkout Ruff
with:
persist-credentials: true
ref: ${{ env.UPSTREAM_BRANCH}}
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
enable-cache: false
# renovate: datasource=github-release-attachments depName=astral-sh/uv
version: "0.13.1"
checksum: "cfe454573412ff7380d15fc7dae5997ab53ffce72fce0a78e91ddbc84001acd2"
- name: Setup git
run: |
git config --global user.name typeshedbot
git config --global user.email '<>'
- name: Sync macOS docstrings
if: ${{ success() }}
env:
FORCE_COLOR: 1
run: |
./scripts/codemod_docstrings.sh
git commit -am "Sync macOS docstrings" --allow-empty
- name: Format the changes
if: ${{ success() }}
env:
FORCE_COLOR: 1
run: |
# Here we just reformat the codemodded stubs so that they are
# consistent with the other typeshed stubs around them.
# Typeshed formats code using black in their CI, so we just invoke
# black on the stubs the same way that typeshed does.
uv run \
--locked \
--only-group=typeshed-formatting \
black "${VENDORED_TYPESHED}/stdlib" \
--config "${VENDORED_TYPESHED}/pyproject.toml" || true
git commit -am "Format codemodded docstrings" --allow-empty
- name: Remove typeshed pyproject.toml file
if: ${{ success() }}
run: |
rm "${VENDORED_TYPESHED}/pyproject.toml"
git commit -am "Remove pyproject.toml file"
- name: Push synced changes upstream
if: ${{ success() }}
run: git push
- name: Apply and push typeshed patches
if: ${{ success() }}
run: |
git apply --directory="${VENDORED_TYPESHED}" crates/ty_vendored/typeshed_patches/*.patch
git add "${VENDORED_TYPESHED}"
git commit -m "Apply typeshed patches"
git push
# Best-effort snapshot update. A failure or timeout here must not prevent PR creation.
update-snapshots:
name: Update snapshots
runs-on: ${{ github.repository == 'astral-sh/ruff' && 'github-ubuntu-24.04-x86_64-32' || 'ubuntu-latest' }}
timeout-minutes: 20
needs: [sync, docstrings-windows, docstrings-macos]
outputs:
base-sha: ${{ steps.checkout.outputs.commit }}
artifact-id: ${{ steps.snapshots.outputs.artifact-id }}
# Don't run the cron job on forks.
# The job will also be skipped if the docstrings-macos job failed.
if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }}
# Allow any failure in this best-effort job without failing the overall workflow.
# This does not automatically cause later steps to run after a failure; a step-level
# `continue-on-error: true` is also required for that (which we do below for one of the steps).
continue-on-error: true
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
name: Checkout Ruff
id: checkout
with:
persist-credentials: false
ref: ${{ env.UPSTREAM_BRANCH}}
- name: "Install Rust toolchain"
run: rustup show
- name: "Install mold"
uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
enable-cache: false
# renovate: datasource=github-release-attachments depName=astral-sh/uv
version: "0.13.1"
checksum: "eadd9d27c134ab5ac1ffd4473b97a741407f3ea238288f03121fc4deddf90220"
- name: Update snapshots
# Snapshot failures are force-passed by cargo-insta, but other tests might still fail
# in this test, causing the step overall to fail. We want to continue to later steps
# anyway here, so that snapshots are updated and pushed even if other tests fail.
continue-on-error: true
env:
# Work around https://github.com/mitsuhiko/insta/pull/924 until its fix is released.
CI: "0"
MDTEST_UPDATE_SNAPSHOTS: "1"
run: uv run --only-dev cargo insta test --accept --all-features --test-runner nextest --disable-nextest-doctest
- name: Prepare snapshot changes
run: |
git add --update
git diff --cached --binary > "${RUNNER_TEMP}/snapshot-updates.patch"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
name: Upload snapshot changes
id: snapshots
with:
name: typeshed-snapshot-updates
path: ${{ runner.temp }}/snapshot-updates.patch
if-no-files-found: error
retention-days: 1
# Create the PR on a separate runner to the job that updates the snapshots.
# This separation provides better security isolation.
create-pr:
name: Create PR
runs-on: ${{ github.repository == 'astral-sh/ruff' && 'github-ubuntu-24.04-x86_64-4' || 'ubuntu-latest' }}
timeout-minutes: 5
needs: [sync, docstrings-windows, docstrings-macos, update-snapshots]
# update-snapshots is deliberately NOT included here: it shouldn't block PR creation if it fails or times out.
if: ${{ always() && needs.sync.result == 'success' && needs.docstrings-windows.result == 'success' && needs.docstrings-macos.result == 'success' && (github.repository == 'astral-sh/ruff' || github.event_name != 'schedule') }}
permissions:
contents: write # to push back to the repository
pull-requests: write # to create a PR
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
name: Checkout Ruff
if: ${{ needs.update-snapshots.outputs.artifact-id != '' }}
with:
# Persist credentials so the later git push can authenticate.
persist-credentials: true
ref: ${{ needs.update-snapshots.outputs.base-sha }}
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
name: Download snapshot changes
if: ${{ needs.update-snapshots.outputs.artifact-id != '' }}
with:
name: typeshed-snapshot-updates
path: ${{ runner.temp }}/snapshot-updates
- name: Commit and push snapshot changes
# Treat generated changes as unreviewed contributions, including when running CI.
# Human review is required before merging.
if: ${{ needs.update-snapshots.outputs.artifact-id != '' }}
run: |
git config --global user.name typeshedbot
git config --global user.email '<>'
# Apply without staging so new files in the patch remain untracked.
git apply --allow-empty "${RUNNER_TEMP}/snapshot-updates/snapshot-updates.patch"
git add --update
if git diff --cached --quiet; then
echo "No snapshot changes to commit"
else
git commit -m "Update snapshots"
git push origin "HEAD:${UPSTREAM_BRANCH}"
fi
- name: Create a PR
run: |
gh pr list --repo "${GITHUB_REPOSITORY}" --head "${UPSTREAM_BRANCH}" --json id --jq length | grep 1 && exit 0 # exit if there is existing pr
gh pr create --repo "${GITHUB_REPOSITORY}" --base main --head "${UPSTREAM_BRANCH}" --title "[ty] Sync vendored typeshed stubs" --body "Close and reopen this PR to trigger CI" --label "ty"
create-issue-on-failure:
name: Create an issue if the typeshed sync failed
runs-on: ${{ github.repository == 'astral-sh/ruff' && 'github-ubuntu-24.04-x86_64-4' || 'ubuntu-latest' }}
needs: [sync, docstrings-windows, docstrings-macos, create-pr]
# update-snapshots is deliberately NOT included here: it shouldn't block PR creation if it fails or times out,
# so there's no need for the bot to create an issue if that happens.
if: ${{ github.repository == 'astral-sh/ruff' && always() && github.event_name == 'schedule' && (needs.sync.result != 'success' || needs.docstrings-windows.result != 'success' || needs.docstrings-macos.result != 'success' || needs.create-pr.result != 'success') }}
permissions:
issues: write # to create an issue on failure
steps:
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.create({
owner: "astral-sh",
repo: "ruff",
title: `Automated typeshed sync failed on ${new Date().toDateString()}`,
body: `Run listed here: https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
labels: ["bug", "ty"],
})