Skip to content

Commit 8fb854c

Browse files
ydshiehydshieh
authored
Run slow v2 (#41914)
* Super * Super * Super * Super --------- Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
1 parent a0bf5a8 commit 8fb854c

12 files changed

+355
-339
lines changed

‎.github/workflows/check_failed_tests.yml‎

Lines changed: 75 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@ on:
66
docker:
77
required: true
88
type: string
9-
start_sha:
10-
required: true
11-
type: string
129
job:
1310
required: true
1411
type: string
@@ -24,7 +21,13 @@ on:
2421
commit_sha:
2522
required: false
2623
type: string
27-
24+
pr_number:
25+
required: false
26+
type: string
27+
outputs:
28+
report:
29+
description: "Content of the report of new failures"
30+
value: ${{ jobs.process_new_failures_with_commit_info.outputs.report }}
2831

2932
env:
3033
HF_HOME: /mnt/cache
@@ -88,27 +91,55 @@ jobs:
8891
echo "PREV_WORKFLOW_RUN_ID=" >> $GITHUB_ENV
8992
fi
9093
91-
if [ -f setup_values/other_workflow_run_id.txt ]; then
92-
echo "OTHER_WORKFLOW_RUN_ID=$(cat setup_values/other_workflow_run_id.txt)" >> $GITHUB_ENV
93-
else
94-
echo "OTHER_WORKFLOW_RUN_ID=" >> $GITHUB_ENV
95-
fi
96-
9794
- name: Update clone
9895
working-directory: /transformers
9996
if: ${{ env.process == 'true' }}
100-
run: git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
97+
run: |
98+
git fetch origin ${{ inputs.commit_sha || github.sha }}
99+
git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
101100
102-
- name: Get target commit
101+
- name: Get `START_SHA`
103102
working-directory: /transformers/utils
104103
if: ${{ env.process == 'true' }}
104+
run: |
105+
echo "START_SHA=${{ inputs.commit_sha || github.sha }}" >> $GITHUB_ENV
106+
107+
# This is used if the CI is triggered from a pull request `self-comment-ci.yml` (after security check is verified)
108+
- name: Extract the base commit on `main` (of the merge commit created by Github) if it is a PR
109+
id: pr_info
110+
if: ${{ env.process == 'true' && inputs.pr_number != '' }}
111+
uses: actions/github-script@v6
112+
with:
113+
script: |
114+
const { data: pr } = await github.rest.pulls.get({
115+
owner: context.repo.owner,
116+
repo: context.repo.repo,
117+
pull_number: ${{ inputs.pr_number }}
118+
});
119+
120+
const { data: merge_commit } = await github.rest.repos.getCommit({
121+
owner: pr.base.repo.owner.login,
122+
repo: pr.base.repo.name,
123+
ref: pr.merge_commit_sha,
124+
});
125+
126+
core.setOutput('merge_commit_base_sha', merge_commit.parents[0].sha);
127+
128+
# Usually, `END_SHA` should be the commit of the last previous workflow run of the **SAME** (scheduled) workflow.
129+
# (This is why we don't need to specify `workflow_id` which would be fetched automatically in the python script.)
130+
- name: Get `END_SHA` from previous CI runs of the same workflow
131+
working-directory: /transformers/utils
132+
if: ${{ env.process == 'true' && inputs.pr_number == '' }}
105133
run: |
106134
echo "END_SHA=$(TOKEN=${{ secrets.ACCESS_REPO_INFO_TOKEN }} python3 -c 'import os; from get_previous_daily_ci import get_last_daily_ci_run_commit; commit=get_last_daily_ci_run_commit(token=os.environ["TOKEN"], workflow_run_id=os.environ["PREV_WORKFLOW_RUN_ID"]); print(commit)')" >> $GITHUB_ENV
107135
108-
- name: Checkout to `start_sha`
109-
working-directory: /transformers
110-
if: ${{ env.process == 'true' }}
111-
run: git fetch && git checkout ${{ inputs.start_sha }}
136+
# However, for workflow runs triggered by `issue_comment` (for pull requests), we want to check against the
137+
# parent commit (on `main`) of the `merge_commit` (dynamically created by GitHub). In this case, the goal is to
138+
# see if a reported failing test is actually ONLY failing on the `merge_commit`.
139+
- name: Set `END_SHA`
140+
if: ${{ env.process == 'true' && inputs.pr_number != '' }}
141+
run: |
142+
echo "END_SHA=${{ steps.pr_info.outputs.merge_commit_base_sha }}" >> $GITHUB_ENV
112143
113144
- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
114145
working-directory: /transformers
@@ -138,7 +169,7 @@ jobs:
138169
- name: Check failed tests
139170
working-directory: /transformers
140171
if: ${{ env.process == 'true' }}
141-
run: python3 utils/check_bad_commit.py --start_commit ${{ inputs.start_sha }} --end_commit ${{ env.END_SHA }} --file ci_results_${{ inputs.job }}/new_failures.json --output_file new_failures_with_bad_commit_${{ inputs.job }}_${{ matrix.run_idx }}.json
172+
run: python3 utils/check_bad_commit.py --start_commit ${{ env.START_SHA }} --end_commit ${{ env.END_SHA }} --file ci_results_${{ inputs.job }}/new_failures.json --output_file new_failures_with_bad_commit_${{ inputs.job }}_${{ matrix.run_idx }}.json
142173

143174
- name: Show results
144175
working-directory: /transformers
@@ -159,6 +190,8 @@ jobs:
159190
if: needs.check_new_failures.outputs.process == 'true'
160191
runs-on:
161192
group: aws-g5-4xlarge-cache
193+
outputs:
194+
report: ${{ steps.set_output.outputs.report }}
162195
container:
163196
image: ${{ inputs.docker }}
164197
options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
@@ -190,18 +223,9 @@ jobs:
190223
191224
- name: Update clone
192225
working-directory: /transformers
193-
run: git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
194-
195-
- name: Process report
196-
shell: bash
197-
working-directory: /transformers
198-
env:
199-
ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
200-
TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}
201-
JOB_NAME: ${{ inputs.job }}
202-
REPORT_REPO_ID: ${{ inputs.report_repo_id }}
203226
run: |
204-
python3 utils/process_bad_commit_report.py
227+
git fetch origin ${{ inputs.commit_sha || github.sha }}
228+
git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
205229
206230
- name: Process report
207231
shell: bash
@@ -218,6 +242,29 @@ jobs:
218242
echo EOF
219243
} >> "$GITHUB_ENV"
220244
245+
# The output is useful if a caller needs more processing, for example, we have a chain
246+
# self-comment-ci.yml -> self-scheduled.yml -> this one (check_failed_tests.yml),
247+
# and `self-comment-ci.yml` needs further processing before sending a GitHub comment to the pull request page.
248+
- name: Show results & Set outputs
249+
id: set_output
250+
working-directory: /transformers
251+
run: |
252+
ls -l new_failures_with_bad_commit.json
253+
cat new_failures_with_bad_commit.json
254+
255+
{
256+
echo 'report<<EOF'
257+
cat new_failures_with_bad_commit.json
258+
echo '' # Force a newline
259+
echo EOF
260+
} >> "$GITHUB_OUTPUT"
261+
262+
- name: Upload artifacts
263+
uses: actions/upload-artifact@v4
264+
with:
265+
name: new_failures_with_bad_commit_${{ inputs.job }}
266+
path: /transformers/new_failures_with_bad_commit.json
267+
221268
- name: Prepare Slack report title
222269
working-directory: /transformers
223270
run: |

‎.github/workflows/get-pr-info.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ on:
3939
PR_MERGE_COMMIT_SHA:
4040
description: "The sha of the merge commit for the pull request (created by GitHub) in the base repository"
4141
value: ${{ jobs.get-pr-info.outputs.PR_MERGE_COMMIT_SHA }}
42+
PR_MERGE_COMMIT_BASE_SHA:
43+
description: "The sha of the parent commit of the the merge commit on the target branch in the base repository"
44+
value: ${{ jobs.get-pr-info.outputs.PR_MERGE_COMMIT_BASE_SHA }}
4245
PR_HEAD_COMMIT_DATE:
4346
description: "The date of the head sha of the pull request branch in the head repository"
4447
value: ${{ jobs.get-pr-info.outputs.PR_HEAD_COMMIT_DATE }}
@@ -74,6 +77,7 @@ jobs:
7477
PR_BASE_REF: ${{ steps.pr_info.outputs.base_ref }}
7578
PR_HEAD_SHA: ${{ steps.pr_info.outputs.head_sha }}
7679
PR_BASE_SHA: ${{ steps.pr_info.outputs.base_sha }}
80+
PR_MERGE_COMMIT_BASE_SHA: ${{ steps.pr_info.outputs.merge_commit_base_sha }}
7781
PR_MERGE_COMMIT_SHA: ${{ steps.pr_info.outputs.merge_commit_sha }}
7882
PR_HEAD_COMMIT_DATE: ${{ steps.pr_info.outputs.head_commit_date }}
7983
PR_MERGE_COMMIT_DATE: ${{ steps.pr_info.outputs.merge_commit_date }}
@@ -122,6 +126,7 @@ jobs:
122126
core.setOutput('base_ref', pr.base.ref);
123127
core.setOutput('head_sha', pr.head.sha);
124128
core.setOutput('base_sha', pr.base.sha);
129+
core.setOutput('merge_commit_base_sha', merge_commit.parents[0].sha);
125130
core.setOutput('merge_commit_sha', pr.merge_commit_sha);
126131
core.setOutput('pr', pr);
127132
@@ -142,6 +147,10 @@ jobs:
142147
date: merge_commit.commit.committer.date
143148
});
144149
150+
console.log('PR Info:', {
151+
pr_info: pr
152+
});
153+
145154
- name: Convert dates to timestamps
146155
id: get_timestamps
147156
run: |

‎.github/workflows/model_jobs.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,9 @@ jobs:
8080
8181
- name: Update clone
8282
working-directory: /transformers
83-
run: git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
83+
run: |
84+
git fetch origin ${{ inputs.commit_sha || github.sha }}
85+
git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
8486
8587
- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
8688
working-directory: /transformers
@@ -174,7 +176,7 @@ jobs:
174176

175177
collated_reports:
176178
name: Collated Reports
177-
if: ${{ always() }}
179+
if: ${{ always() && inputs.runner_type != '' }}
178180
needs: run_models_gpu
179181
uses: huggingface/transformers/.github/workflows/collated-reports.yml@main
180182
with:

‎.github/workflows/push-important-models.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,5 +153,5 @@ jobs:
153153
ci_event: push
154154
report_repo_id: hf-internal-testing/transformers_ci_push
155155
commit_sha: ${{ github.sha }}
156-
models: ${{ needs.get_modified_models.outputs.matrix }}
156+
subdirs: ${{ needs.get_modified_models.outputs.matrix }}
157157
secrets: inherit

0 commit comments

Comments
 (0)