-
Notifications
You must be signed in to change notification settings - Fork 12.3k
Expand file tree
/
Copy pathtest_github_workflows.py
More file actions
356 lines (292 loc) · 12.6 KB
/
Copy pathtest_github_workflows.py
File metadata and controls
356 lines (292 loc) · 12.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
"""Static checks for repository GitHub Actions workflows."""
from __future__ import annotations
import os
import re
import subprocess
import sys
from pathlib import Path
import yaml
from tests.conftest import requires_bash
REPO_ROOT = Path(__file__).resolve().parent.parent
WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows"
# Match both the dedicated-step form (` uses: x@sha`) and the
# inline shorthand (` - uses: x@sha`) used in catalog-assign.yml.
USES_RE = re.compile(r"^\s*(?:-\s*)?uses:\s*(?P<ref>\S+)", re.MULTILINE)
PINNED_SHA_RE = re.compile(r"@[0-9a-f]{40}$", re.IGNORECASE)
PUBLISH_WORKFLOW = WORKFLOWS_DIR / "publish-pypi.yml"
PUBLISH_VALIDATION_STEPS = (
"Verify tag format",
"Verify tag matches package version",
)
COMMUNITY_SUBMISSION_WORKFLOWS = (
(
"bundle",
"bundle-submission",
"bundles/catalog.community.json",
"docs/community/bundles.md",
"Modify only `bundles/catalog.community.json`",
),
(
"extension",
"extension-submission",
"extensions/catalog.community.json",
"docs/community/extensions.md",
"Do not modify any other files",
),
(
"preset",
"preset-submission",
"presets/catalog.community.json",
"docs/community/presets.md",
"Do not modify any other files",
),
)
def _publish_workflow_steps() -> dict[str, dict[str, object]]:
workflow = yaml.safe_load(PUBLISH_WORKFLOW.read_text(encoding="utf-8"))
return {step["name"]: step for step in workflow["jobs"]["build"]["steps"]}
def _run_publish_validation_step(
step_name: str, tag: str, working_directory: Path
) -> subprocess.CompletedProcess[str]:
step = _publish_workflow_steps()[step_name]
env = os.environ.copy()
env["TAG"] = tag
env["PATH"] = f"{Path(sys.executable).parent}{os.pathsep}{env['PATH']}"
return subprocess.run(
["bash", "-euo", "pipefail", "-c", step["run"]],
cwd=working_directory,
env=env,
capture_output=True,
text=True,
check=False,
)
def _write_project_version(working_directory: Path, version: str) -> None:
(working_directory / "pyproject.toml").write_text(
f'[project]\nversion = "{version}"\n', encoding="utf-8"
)
def _create_pull_request_allowed_files(source_text: str) -> list[str]:
create_pr_match = re.search(
r"(?m)^ create-pull-request:\n(?P<body>(?:^ [^\n]*\n?)+)",
source_text,
)
assert create_pr_match is not None
allowed_files_match = re.search(
r"(?m)^ allowed-files:\n(?P<files>(?:^ - [^\n]+\n?)+)",
create_pr_match.group("body"),
)
assert allowed_files_match is not None
return [
line.strip().removeprefix("- ")
for line in allowed_files_match.group("files").splitlines()
if line.strip()
]
def test_github_actions_are_pinned_to_full_commit_shas():
unpinned_refs = []
workflows = sorted(
list(WORKFLOWS_DIR.glob("*.yml")) + list(WORKFLOWS_DIR.glob("*.yaml"))
)
assert workflows
for workflow in workflows:
workflow_text = workflow.read_text(encoding="utf-8")
for match in USES_RE.finditer(workflow_text):
uses_ref = match.group("ref")
if uses_ref.startswith(("./", "../")):
continue
if PINNED_SHA_RE.search(uses_ref):
continue
unpinned_refs.append(f"{workflow.relative_to(REPO_ROOT)}: {uses_ref}")
assert unpinned_refs == []
def test_publish_tag_validation_uses_environment_variable():
steps = _publish_workflow_steps()
for step_name in PUBLISH_VALIDATION_STEPS:
step = steps[step_name]
assert step["env"]["TAG"] == "${{ inputs.tag }}"
assert "${{ inputs.tag }}" not in step["run"]
@requires_bash
def test_publish_tag_validation_accepts_valid_tag(tmp_path):
_write_project_version(tmp_path, "1.2.3")
for step_name in PUBLISH_VALIDATION_STEPS:
result = _run_publish_validation_step(step_name, "v1.2.3", tmp_path)
assert result.returncode == 0, result.stderr
@requires_bash
def test_publish_tag_validation_rejects_invalid_tag(tmp_path):
for invalid_tag in ("1.2.3", "v1.2", "v1.2.3-rc1"):
result = _run_publish_validation_step(
"Verify tag format", invalid_tag, tmp_path
)
assert result.returncode != 0
assert "is not a valid release tag" in result.stdout
injected_file = tmp_path / "interpolated"
injected_tag = f'v1.2.3"; touch "{injected_file}"; #'
result = _run_publish_validation_step("Verify tag format", injected_tag, tmp_path)
assert result.returncode != 0
assert not injected_file.exists()
@requires_bash
def test_publish_tag_validation_rejects_version_mismatch(tmp_path):
_write_project_version(tmp_path, "1.2.3")
result = _run_publish_validation_step(
"Verify tag matches package version", "v1.2.4", tmp_path
)
assert result.returncode != 0
assert "does not match pyproject.toml version" in result.stdout
def test_pinned_action_ref_accepts_uppercase_hex_sha():
assert PINNED_SHA_RE.search(
"actions/example@0123456789ABCDEF0123456789ABCDEF01234567"
)
def test_community_submission_automation_is_wired_to_allowed_files():
assignment = WORKFLOWS_DIR / "catalog-assign.yml"
assignment_text = assignment.read_text(encoding="utf-8")
for workflow, label, catalog_file, docs_file, instruction in (
COMMUNITY_SUBMISSION_WORKFLOWS
):
source = WORKFLOWS_DIR / f"add-community-{workflow}.md"
compiled = WORKFLOWS_DIR / f"add-community-{workflow}.lock.yml"
assert source.is_file()
assert compiled.is_file()
source_text = source.read_text(encoding="utf-8")
compiled_text = compiled.read_text(encoding="utf-8")
assert f"names: [{label}]" in source_text
assert catalog_file in source_text
assert docs_file in source_text
assert instruction in source_text
assert _create_pull_request_allowed_files(source_text) == [
catalog_file,
docs_file,
]
assert f'"allowed_files":["{catalog_file}","{docs_file}"]' in compiled_text
assert label in assignment_text
# Full clauses from the catalog download-URL checks (issue #4185). Assert the
# complete sentences so independent keywords cannot drift apart.
_CATALOG_DOWNLOAD_URL_CLAUSES = (
(
"The download URL MUST belong to the submitted repository\n"
" (`https://github.com/<owner>/<repo>/...` with the same `<owner>/<repo>` as\n"
" the Repository URL). Reject URLs for any other GitHub repository."
),
(
"If the download URL path contains `releases/latest/`, reject with an\n"
" explanation — this URL is floating and not acceptable. Mark this pinning\n"
" check failed and skip the HTTP request for this URL, then continue the\n"
" remaining validations."
),
(
"The `<tag>` segment in the URL MUST correspond to the submitted version.\n"
" Accept `vX.Y.Z`, `X.Y.Z`, and scoped tags whose version suffix matches\n"
" (for example `aide-v1.0.0` for version `1.0.0`). Reject a tag whose\n"
" embedded semver does not equal the submitted version."
),
(
"Only after all pinning checks pass, fetch the download URL and perform the\n"
" remaining artifact checks:\n"
" - Verify the URL returns HTTP 200.\n"
" - If `sha256` is included, verify it matches the downloaded archive. Requiring\n"
" `sha256` on every catalog entry is follow-up work and MUST NOT fail this\n"
" check when the field is absent."
),
)
def test_community_submission_workflows_require_tag_pinned_download_urls():
"""Catalog agents must reject floating releases/latest URLs (issue #4185)."""
for workflow, *_ in COMMUNITY_SUBMISSION_WORKFLOWS:
source_text = (WORKFLOWS_DIR / f"add-community-{workflow}.md").read_text(
encoding="utf-8"
)
assert "should follow the pattern" not in source_text.lower()
for clause in _CATALOG_DOWNLOAD_URL_CLAUSES:
assert clause in source_text, f"missing clause in {workflow}: {clause!r}"
if workflow == "bundle":
assert (
"`https://github.com/<owner>/<repo>/releases/download/<tag>/<asset>.zip`."
in source_text
)
assert "archive/refs/tags/" not in source_text
else:
assert (
"`https://github.com/<owner>/<repo>/archive/refs/tags/<tag>.zip`"
in source_text
)
assert (
"`https://github.com/<owner>/<repo>/releases/download/<tag>/<asset>.zip`"
in source_text
)
def test_community_submission_allowed_files_do_not_include_other_catalogs_or_docs():
allowed_by_workflow = {
workflow: set(
_create_pull_request_allowed_files(
(WORKFLOWS_DIR / f"add-community-{workflow}.md").read_text(
encoding="utf-8"
)
)
)
for workflow, *_ in COMMUNITY_SUBMISSION_WORKFLOWS
}
workflow_allowed_files = list(allowed_by_workflow.items())
for index, (workflow, allowed_files) in enumerate(workflow_allowed_files):
for other_workflow, other_allowed_files in workflow_allowed_files[index + 1 :]:
overlapping_files = allowed_files & other_allowed_files
assert overlapping_files == set(), (
f"{workflow} and {other_workflow} share allowed files: "
f"{sorted(overlapping_files)}"
)
def _frontmatter(source_text: str) -> dict:
if not source_text.startswith("---"):
raise AssertionError("workflow source is missing YAML frontmatter")
_, frontmatter, _ = source_text.split("---", 2)
return yaml.safe_load(frontmatter)
def test_community_submission_threat_detection_is_fail_closed():
for workflow, *_ in COMMUNITY_SUBMISSION_WORKFLOWS:
source = WORKFLOWS_DIR / f"add-community-{workflow}.md"
compiled = WORKFLOWS_DIR / f"add-community-{workflow}.lock.yml"
assert source.is_file()
assert compiled.is_file()
safe_outputs = _frontmatter(source.read_text(encoding="utf-8")).get(
"safe-outputs", {}
)
threat_detection = safe_outputs.get("threat-detection")
assert threat_detection is not None, (
f"add-community-{workflow}.md must configure "
"safe-outputs.threat-detection"
)
assert threat_detection.get("continue-on-error") is False, (
f"add-community-{workflow}.md must set threat-detection "
"continue-on-error: false so detections block safe outputs"
)
compiled_text = compiled.read_text(encoding="utf-8")
assert 'GH_AW_DETECTION_CONTINUE_ON_ERROR: "false"' in compiled_text, (
f"add-community-{workflow}.lock.yml must compile threat detection "
"in fail-closed mode"
)
assert (
"process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'"
in compiled_text
), (
f"add-community-{workflow}.lock.yml is missing the detection "
"continue-on-error gate"
)
def test_bug_test_workflow_provisions_python_dependencies():
source = WORKFLOWS_DIR / "bug-test.md"
compiled = WORKFLOWS_DIR / "bug-test.lock.yml"
assert source.is_file()
assert compiled.is_file()
source_text = source.read_text(encoding="utf-8")
compiled_text = compiled.read_text(encoding="utf-8")
setup_uv = (
"astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1"
)
setup_python = (
"actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0"
)
assert " - pypi.org" in source_text
assert " - files.pythonhosted.org" in source_text
assert setup_uv in source_text
assert setup_python in source_text
assert 'run: uv pip install --system -e ".[test]"' in source_text
assert '"pypi.org"' in compiled_text
assert '"files.pythonhosted.org"' in compiled_text
checkout_index = compiled_text.index("- name: Checkout repository")
uv_index = compiled_text.index("- name: Setup uv")
python_index = compiled_text.index("- name: Set up Python")
sync_index = compiled_text.index("- name: Install Python test dependencies")
agent_index = compiled_text.index("- name: Execute GitHub Copilot CLI")
assert checkout_index < uv_index < python_index < sync_index < agent_index
assert setup_uv in compiled_text
assert setup_python in compiled_text
assert 'run: uv pip install --system -e ".[test]"' in compiled_text