-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathvalidators.py
More file actions
704 lines (604 loc) · 27.7 KB
/
Copy pathvalidators.py
File metadata and controls
704 lines (604 loc) · 27.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
"""Run real tools over a unit and report what they say.
A benchmark that scores with an LLM judge measures the judge. These validators
are the actual tools an engineer would run, so a task is passed or failed by
`helm`, `hadolint` and `terraform` rather than by an opinion.
Two signals come out of each run and they are not the same thing:
- **structural validity**: the tool accepts the unit at all. This is the exit
code, once thresholds are set so that style warnings do not count as failure.
Left at its default, `hadolint` rejects almost every real Dockerfile, which
would make the score meaningless.
- **diagnostics**: the specific rules that fired, by code. Removing `USER` from a
Dockerfile is not a structural break, so only a per-rule signal can tell whether
a repair restored it.
Every run is offline and hermetic: no network, tool caches redirected into a
temporary directory, and a timeout, because `helm template` evaluates Go templates
that can loop.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import sys
import tempfile
from dataclasses import dataclass, field
DEFAULT_TIMEOUT = 30
@dataclass(frozen=True)
class Verdict:
"""What a tool said about one unit."""
tool: str
ok: bool
exit_code: int
diagnostics: tuple[str, ...] = ()
output: str = ""
timed_out: bool = False
unavailable: bool = False
@property
def summary(self) -> str:
if self.unavailable:
return f"{self.tool}: not installed"
if self.timed_out:
return f"{self.tool}: timed out"
state = "ok" if self.ok else f"failed ({self.exit_code})"
codes = f" [{', '.join(self.diagnostics)}]" if self.diagnostics else ""
return f"{self.tool}: {state}{codes}"
def resolve(tool: str) -> str | None:
"""Absolute path to a tool, including ones installed alongside the interpreter.
`shutil.which` alone misses tools installed as scripts in a virtualenv, because
running `.venv/bin/python` does not put `.venv/bin` on PATH. ansible-lint lives
there, and without this the validator reported it as missing and the class was
skipped entirely.
"""
found = shutil.which(tool)
if found:
return found
beside = os.path.join(os.path.dirname(sys.executable), tool)
return beside if os.access(beside, os.X_OK) else None
def available(tool: str) -> bool:
return resolve(tool) is not None
def _hermetic_env(scratch: str) -> dict[str, str]:
"""Redirect tool caches without breaking the tools themselves.
HOME is deliberately left alone. Overriding it looks like good hygiene and
silently disables every version-manager shim: `terraform` is frequently tfenv,
which reads `$HOME/.config/tfenv/version` to find the real binary and dies
without it. The validator then never ran terraform at all while still reporting
success, which is the most dangerous failure a benchmark can have. Only the
per-tool cache and config paths are redirected, which is what actually needs
isolating.
"""
return {
**os.environ,
"TMPDIR": scratch,
"HELM_CACHE_HOME": os.path.join(scratch, "helm-cache"),
"HELM_CONFIG_HOME": os.path.join(scratch, "helm-config"),
"HELM_DATA_HOME": os.path.join(scratch, "helm-data"),
"TF_DATA_DIR": os.path.join(scratch, "terraform-data"),
"TF_IN_AUTOMATION": "1",
"CHECKPOINT_DISABLE": "1", # terraform's version check phones home otherwise
"NO_COLOR": "1",
}
def run_tool(
command: list[str],
directory: str,
timeout: int = DEFAULT_TIMEOUT,
) -> tuple[int, str, str, bool]:
"""Run a command in `directory`. Returns (exit code, stdout, stderr, timed out)."""
resolved = resolve(command[0])
if resolved is None:
return -2, "", "tool not found", False
command = [resolved, *command[1:]]
with tempfile.TemporaryDirectory(prefix="devopsbench-") as scratch:
try:
completed = subprocess.run(
command,
cwd=directory,
capture_output=True,
text=True,
timeout=timeout,
env=_hermetic_env(scratch),
check=False,
)
except subprocess.TimeoutExpired:
return -1, "", "timed out", True
except FileNotFoundError:
return -2, "", "tool not found", False
return completed.returncode, completed.stdout, completed.stderr, False
def hadolint(dockerfile_path: str, directory: str, timeout: int = DEFAULT_TIMEOUT) -> Verdict:
"""Lint a Dockerfile.
`--failure-threshold error` is deliberate. By default hadolint exits non-zero
on warnings, and warnings fire on nearly every real Dockerfile (unpinned apk
packages alone), so the default exit code carries no information about whether
the file is broken. Rule codes are collected separately.
"""
if not available("hadolint"):
return Verdict("hadolint", False, -2, unavailable=True)
code, out, err, timed_out = run_tool(
["hadolint", "--format", "json", "--failure-threshold", "error", dockerfile_path],
directory,
timeout,
)
if timed_out:
return Verdict("hadolint", False, code, timed_out=True)
diagnostics: list[str] = []
try:
for item in json.loads(out or "[]"):
diagnostics.append(str(item.get("code")))
except json.JSONDecodeError:
diagnostics = []
return Verdict(
"hadolint",
ok=code == 0,
exit_code=code,
diagnostics=tuple(diagnostics),
output=(err or out)[-2000:],
)
def helm_template(chart_dir: str, timeout: int = DEFAULT_TIMEOUT) -> tuple[Verdict, str]:
"""Render a chart, returning the verdict and the rendered manifests.
The full render is returned rather than just a verdict because policy rules
apply to manifests, not to templates: `{{ toYaml .Values.resources }}` in a
template says nothing about whether any resources end up being set.
"""
if not available("helm"):
return Verdict("helm template", False, -2, unavailable=True), ""
code, out, err, timed_out = run_tool(
["helm", "template", "bench-release", "."], chart_dir, timeout
)
if timed_out:
return Verdict("helm template", False, code, timed_out=True), ""
verdict = Verdict("helm template", ok=code == 0, exit_code=code,
output=(err or out)[-2000:])
return verdict, (out if code == 0 else "")
def helm_lint(chart_dir: str, timeout: int = DEFAULT_TIMEOUT) -> Verdict:
"""Lint a chart, which also applies Kubernetes schema checks.
Stricter than `helm template`: a Deployment without `selector.matchLabels` is
rejected here and rendered happily there. Many real charts fail for reasons
that predate any mutation, which is why task generation requires the original
to pass before a mutation counts.
"""
if not available("helm"):
return Verdict("helm lint", False, -2, unavailable=True)
code, out, err, timed_out = run_tool(["helm", "lint", "."], chart_dir, timeout)
if timed_out:
return Verdict("helm lint", False, code, timed_out=True)
diagnostics = tuple(
line.split("]", 1)[0].lstrip("[")
for line in (out or "").splitlines()
if line.startswith("[")
)
return Verdict(
"helm lint",
ok=code == 0,
exit_code=code,
diagnostics=diagnostics,
output=(err or out)[-2000:],
)
# actionlint diagnostic kinds that mean the workflow is wrong, as opposed to
# outdated or unfashionable. Measured over 40 real workflows: 29 were rejected, and
# the complaints were dominated by `action` (63 times: `actions/checkout@v2` is old
# but perfectly valid) and `shellcheck` (42 times: shell style inside `run:`).
# Gating on those would reject three quarters of the real world for reasons no
# repair task is about.
ACTIONLINT_VALIDITY_KINDS = frozenset({
"syntax-check",
"expression",
"matrix",
"events",
"workflow-call",
"job-needs",
"id",
"env-var",
"shell-name",
"glob",
})
def actionlint(workflow_paths: list[str], directory: str,
timeout: int = DEFAULT_TIMEOUT) -> Verdict:
"""Lint GitHub Actions workflows.
Two things here are deliberate.
Every workflow file is passed explicitly. Left to discover them itself,
actionlint looks for a git repository and exits 3 with "no project was found" on
a plain directory, identically for a sound workflow and a broken one.
The shellcheck and pyflakes integrations are switched off. actionlint shells out
to them when they happen to be installed, so leaving them on would make the same
workflow pass on one machine and fail on another, which no benchmark can afford.
"""
if not available("actionlint"):
return Verdict("actionlint", False, -2, unavailable=True)
if not workflow_paths:
return Verdict("actionlint", False, -3, diagnostics=("no_workflow_file",))
code, out, err, timed_out = run_tool(
["actionlint", "-no-color", "-shellcheck=", "-pyflakes=", *workflow_paths],
directory,
timeout,
)
if timed_out:
return Verdict("actionlint", False, code, timed_out=True)
kinds = tuple(
line.rsplit("[", 1)[1].rstrip("]")
for line in (out or "").splitlines()
if line.rstrip().endswith("]") and "[" in line
)
invalid = [kind for kind in kinds if kind in ACTIONLINT_VALIDITY_KINDS]
return Verdict("actionlint", ok=not invalid, exit_code=code,
diagnostics=kinds, output=(out or err)[-2000:])
def ansible_lint(role_dir: str, timeout: int = 120) -> Verdict:
"""Lint an Ansible role.
`--profile min` is the whole point. On its default `production` profile
ansible-lint rejected 19 of 20 real roles, almost entirely on naming and
fully-qualified-name style, which says nothing about whether the role works.
`min` is syntax errors and unloadable files, which is what a validity gate wants,
and it rejected none of the same 20.
`--offline` stops it reaching for collections from Galaxy.
"""
if not available("ansible-lint"):
return Verdict("ansible-lint", False, -2, unavailable=True)
code, out, err, timed_out = run_tool(
["ansible-lint", "--nocolor", "--offline", "--profile", "min", "."],
role_dir,
timeout,
)
if timed_out:
return Verdict("ansible-lint", False, code, timed_out=True)
diagnostics = tuple(
line.split(":", 1)[0].strip()
for line in (out or "").splitlines()
if line and not line.startswith((" ", "\t")) and ":" in line
and line.split(":", 1)[0].strip().replace("-", "").replace("[", "").isalnum()
)
return Verdict("ansible-lint", ok=code == 0, exit_code=code,
diagnostics=diagnostics[:10], output=(out or err)[-2000:])
def compose_config(compose_paths: list[str], directory: str,
timeout: int = DEFAULT_TIMEOUT) -> Verdict:
"""Validate a Compose file.
`docker compose config` parses and resolves the file client-side, so it needs no
daemon, and it reports precise schema errors such as
`services.web.ports must be a array`.
"""
if not available("docker"):
return Verdict("docker compose config", False, -2, unavailable=True)
if not compose_paths:
return Verdict("docker compose config", False, -3,
diagnostics=("no_compose_file",))
arguments = []
for path in compose_paths:
arguments += ["-f", path]
code, out, err, timed_out = run_tool(
["docker", "compose", *arguments, "config"], directory, timeout
)
if timed_out:
return Verdict("docker compose config", False, code, timed_out=True)
return Verdict("docker compose config", ok=code == 0, exit_code=code,
output=(err or out)[-2000:])
def manifest_structure(files: dict[str, str]) -> Verdict:
"""Structural validation of Kubernetes manifests, offline and cluster-free."""
from .manifests import inspect
problems = inspect(files)
return Verdict(
"manifest structure",
ok=not problems,
exit_code=len(problems),
diagnostics=tuple(sorted({p.code for p in problems})),
output="; ".join(str(p) for p in problems)[-2000:],
)
# Diagnostics terraform emits for input it cannot parse. Anything else on a
# non-zero exit means the tool itself failed, not the module.
TF_PARSE_ERRORS = (
"Unclosed configuration block",
"Invalid block definition",
"Invalid single-argument block definition",
"Argument or block definition required",
"Invalid expression",
"Unbalanced",
"Missing required argument",
"Unsupported argument",
"Invalid reference",
"Reference to undeclared input variable",
"Duplicate",
)
def terraform_syntax(module_dir: str, timeout: int = DEFAULT_TIMEOUT) -> Verdict:
"""Check that a Terraform module parses and its references resolve.
`terraform validate` needs `terraform init` before it can check providers, but
it parses first and reports syntax and reference errors without any network
access, so it is used here and a provider complaint is treated as success. If
the exit code is non-zero for a reason that is not a recognised terraform
diagnostic, the tool failed rather than the module, and that is reported as an
error instead of quietly passing.
"""
if not available("terraform"):
return Verdict("terraform validate", False, -2, unavailable=True)
code, out, err, timed_out = run_tool(
["terraform", "validate", "-no-color"], module_dir, timeout
)
if timed_out:
return Verdict("terraform validate", False, code, timed_out=True)
combined = f"{out}\n{err}"
if code == 0:
return Verdict("terraform validate", ok=True, exit_code=code, output=combined[-2000:])
# A missing provider means parsing succeeded; that is as far as we can get
# offline, so it counts as valid.
if "Missing required provider" in combined or "provider registry" in combined:
return Verdict("terraform validate", ok=True, exit_code=code, output=combined[-2000:])
matched = [name for name in TF_PARSE_ERRORS if name in combined]
if matched:
return Verdict("terraform validate", ok=False, exit_code=code,
diagnostics=tuple(matched), output=combined[-2000:])
return Verdict("terraform validate", ok=False, exit_code=code,
diagnostics=("tool_error",), output=combined[-2000:],
unavailable=True)
@dataclass
class Report:
"""Every verdict for one unit.
`verdicts` decide pass or fail. `advisory` are recorded and reported but never
gate, for tools whose strictness would exclude sound units.
"""
verdicts: list[Verdict] = field(default_factory=list)
advisory: list[Verdict] = field(default_factory=list)
@property
def ok(self) -> bool:
"""Structurally valid according to every tool that ran."""
applicable = [v for v in self.verdicts if not v.unavailable]
return bool(applicable) and all(v.ok for v in applicable)
@property
def diagnostics(self) -> set[str]:
return {code for v in self.verdicts + self.advisory for code in v.diagnostics}
def __str__(self) -> str:
return "; ".join(v.summary for v in self.verdicts + self.advisory)
def read_files(directory: str) -> dict[str, str]:
"""Read a materialised unit back, for rules that work on source rather than output."""
files: dict[str, str] = {}
for root, _, names in os.walk(directory):
for name in names:
path = os.path.join(root, name)
relative = os.path.relpath(path, directory)
try:
with open(path, encoding="utf-8", errors="replace") as handle:
files[relative] = handle.read()
except OSError:
continue
return files
def policy_verdict(unit_type: str, files: dict[str, str]) -> Verdict:
"""Our own rules, for defects the tools do not report.
Always advisory: a Dockerfile with no USER builds and runs, so a policy finding
must never make a unit count as invalid. `ok` reflects whether anything fired
only so the summary reads sensibly.
"""
from . import policy
codes = policy.check(unit_type, files)
return Verdict("policy", ok=not codes, exit_code=len(codes), diagnostics=codes)
def validate(unit_type: str, directory: str, timeout: int = DEFAULT_TIMEOUT,
tier: str = "offline", runner=None) -> Report:
"""Run every validator that applies to a unit type.
The offline tier is the default and needs nothing but the tools installed. The
network tier adds validators that download or execute, and is opt-in for exactly
that reason.
"""
report = Report()
if tier == "network" and unit_type == "terraform_module":
files = read_files(directory)
report.verdicts.append(terraform_semantic(files, runner, max(timeout, 300)))
report.advisory.append(policy_verdict(unit_type, files))
return report
if unit_type == "dockerfile":
names = [
name for name in sorted(os.listdir(directory))
if name.lower().startswith(("dockerfile", "containerfile"))
]
for name in names or ["Dockerfile"]:
report.verdicts.append(hadolint(name, directory, timeout))
report.advisory.append(policy_verdict(unit_type, read_files(directory)))
elif unit_type == "helm_chart":
# `helm template` gates; `helm lint` is stricter and rejects a third of real
# charts for reasons that predate any mutation (a Deployment with no
# selector, say), so it is recorded but not allowed to gate.
verdict, rendered = helm_template(directory, timeout)
report.verdicts.append(verdict)
report.advisory.append(helm_lint(directory, timeout))
if rendered:
report.advisory.append(policy_verdict(unit_type, {"rendered.yaml": rendered}))
elif unit_type == "terraform_module":
report.verdicts.append(terraform_syntax(directory, timeout))
report.advisory.append(policy_verdict(unit_type, read_files(directory)))
elif unit_type == "workflow":
files = read_files(directory)
# Any YAML in the unit, not only files under .github/workflows/. actionlint
# accepts a workflow at any path, and requiring the canonical directory made
# the validator report every real workflow as already broken because it found
# no files to lint at all.
paths = sorted(
name for name in files if name.lower().endswith((".yml", ".yaml"))
)
report.verdicts.append(actionlint(paths, directory, timeout))
report.advisory.append(policy_verdict(unit_type, files))
elif unit_type == "manifest_set":
files = read_files(directory)
report.verdicts.append(manifest_structure(files))
report.advisory.append(policy_verdict(unit_type, files))
elif unit_type == "ansible_role":
report.verdicts.append(ansible_lint(directory, max(timeout, 120)))
report.advisory.append(policy_verdict(unit_type, read_files(directory)))
elif unit_type == "compose":
files = read_files(directory)
paths = sorted(
name for name in files if name.lower().endswith((".yml", ".yaml"))
)
report.verdicts.append(compose_config(paths, directory, timeout))
report.advisory.append(policy_verdict(unit_type, files))
else:
raise ValueError(f"no validators for {unit_type} yet")
return report
# --- network tier ----------------------------------------------------------
def terraform_semantic(files: dict[str, str], runner=None,
timeout: int = 300) -> Verdict:
"""Initialise providers, then validate, which is the only way to see references.
Offline, `terraform validate` parses and then stops at "Missing required provider"
before checking anything semantic, so a variable used but never declared is
invisible. After `terraform init -backend=false` the same command reports
"Reference to undeclared input variable", which is what makes semantic Terraform
tasks possible at all.
`-backend=false` matters: it fetches providers without touching remote state.
"""
from .runners import LocalRunner
runner = runner or LocalRunner()
# init and validate run in one invocation because validate needs the .terraform/
# directory init leaves behind, and each run() call gets a fresh directory.
result = runner.run(
["sh", "-c",
"terraform init -backend=false -input=false -no-color > /tmp/init.log 2>&1 "
"|| { echo INIT_FAILED; cat /tmp/init.log; exit 90; }; "
"terraform validate -no-color"],
files,
timeout,
)
if result.timed_out:
return Verdict("terraform validate (initialised)", False, -1, timed_out=True)
if result.exit_code == 90:
# A provider that cannot be fetched is an environment problem, not a defect in
# the module, so it must not read as a failing unit.
return Verdict("terraform init", ok=False, exit_code=90,
diagnostics=("init_failed",), output=result.output[-2000:],
unavailable=True)
matched = [name for name in TF_PARSE_ERRORS if name in result.output]
return Verdict(
"terraform validate (initialised)",
ok=result.exit_code == 0,
exit_code=result.exit_code,
diagnostics=tuple(matched) or (() if result.exit_code == 0 else ("error",)),
output=result.output[-2000:],
)
def docker_build(files: dict[str, str], runner=None, timeout: int = 600) -> Verdict:
"""Build the image, which is the only way to know a Dockerfile works.
Every `RUN` line in the unit executes, and the unit came from a repository nobody
vetted, so this refuses to run anywhere that does not isolate.
"""
from .runners import LocalRunner, require_isolation
runner = runner or LocalRunner()
command = ["docker", "build", "--no-cache", "-t", "devopsbench-probe", "."]
require_isolation(runner, command)
built = runner.run(command, files, timeout)
if built.timed_out:
return Verdict("docker build", False, -1, timed_out=True)
return Verdict("docker build", ok=built.exit_code == 0,
exit_code=built.exit_code, output=built.output[-2000:])
# --- self-test -------------------------------------------------------------
# A known-good and a known-broken unit per class. These exist because a validator
# that silently stops working reports everything as valid, and a benchmark built on
# it would score every model as perfect. Overriding HOME once disabled the tfenv
# shim so terraform never ran, and both a sound module and a module with an
# unclosed block came back "ok". Nothing here is trusted until it has just
# distinguished these two fixtures.
SELFTEST_FIXTURES: dict[str, tuple[dict[str, str], dict[str, str]]] = {
"dockerfile": (
{"Dockerfile": "FROM alpine:3.20\nRUN true\nCMD [\"sh\"]\n"},
{"Dockerfile": "FORM alpine:3.20\nRUN true\n"},
),
"helm_chart": (
{
"Chart.yaml": "apiVersion: v2\nname: selftest\nversion: 1.0.0\n",
"values.yaml": "replicas: 1\n",
"templates/cm.yaml": (
"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: selftest\n"
"data:\n {{- if .Values.replicas }}\n n: \"{{ .Values.replicas }}\"\n"
" {{- end }}\n"
),
},
{
"Chart.yaml": "apiVersion: v2\nname: selftest\nversion: 1.0.0\n",
"values.yaml": "replicas: 1\n",
"templates/cm.yaml": (
"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: selftest\n"
"data:\n {{- if .Values.replicas }}\n n: \"{{ .Values.replicas }}\"\n"
),
},
),
"terraform_module": (
{
"variables.tf": 'variable "name" {\n type = string\n}\n',
"main.tf": 'resource "aws_s3_bucket" "b" {\n bucket = var.name\n}\n',
},
{
"variables.tf": 'variable "name" {\n type = string\n}\n',
"main.tf": 'resource "aws_s3_bucket" "b" {\n bucket = var.name\n',
},
),
"workflow": (
{".github/workflows/ci.yml":
"name: CI\non:\n push:\n branches: [main]\n"
"jobs:\n build:\n runs-on: ubuntu-latest\n"
" steps:\n - run: make test\n"},
{".github/workflows/ci.yml":
"name: CI\non:\n push:\n"
"jobs:\n build:\n runs-on: ubuntu-latest\n"
" steps:\n - run: echo hi\n"
" if: ${{ github.event_name = 'push' }}\n"},
),
"ansible_role": (
{
"tasks/main.yml":
"- name: Install nginx\n ansible.builtin.package:\n"
" name: nginx\n state: present\n",
"defaults/main.yml": "nginx_port: 80\n",
},
{
"tasks/main.yml":
"- name: broken\n ansible.builtin.package:\n name: [unclosed\n",
"defaults/main.yml": "nginx_port: 80\n",
},
),
"compose": (
{"docker-compose.yml":
"services:\n web:\n image: nginx:1.27-alpine\n"
" ports:\n - \"8080:80\"\n"},
{"docker-compose.yml":
"services:\n web:\n image: nginx:1.27-alpine\n"
" ports: \"not-a-list\"\n"},
),
"manifest_set": (
{
"deployment.yaml":
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\n"
"spec:\n selector:\n matchLabels:\n app: api\n"
" template:\n metadata:\n labels:\n app: api\n"
" spec:\n containers:\n - name: api\n image: api:1\n",
"service.yaml":
"apiVersion: v1\nkind: Service\nmetadata:\n name: api\n"
"spec:\n ports:\n - port: 80\n",
},
{
"deployment.yaml":
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\n"
"spec:\n template:\n spec:\n containers:\n"
" - name: api\n image: api:1\n",
"service.yaml":
"apiVersion: v1\nkind: Service\nmetadata:\n name: api\n"
"spec:\n ports:\n - port: 80\n",
},
),
}
@dataclass(frozen=True)
class SelfTest:
unit_type: str
passed: bool
reason: str = ""
def selftest(unit_type: str, timeout: int = DEFAULT_TIMEOUT,
tier: str = "offline", runner=None) -> SelfTest:
"""Prove the validators for a class accept a good unit and reject a broken one."""
import pathlib
fixtures = SELFTEST_FIXTURES.get(unit_type)
if fixtures is None:
return SelfTest(unit_type, False, "no fixtures")
good, bad = fixtures
def run(files: dict[str, str]) -> Report:
with tempfile.TemporaryDirectory(prefix="selftest-") as directory:
for name, content in files.items():
target = pathlib.Path(directory, name)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content)
return validate(unit_type, directory, timeout, tier=tier, runner=runner)
good_report, bad_report = run(good), run(bad)
missing = [v.tool for v in good_report.verdicts if v.unavailable]
if missing:
return SelfTest(unit_type, False, f"tool unavailable or failing: {missing}")
if not good_report.ok:
return SelfTest(unit_type, False, f"rejected a valid unit: {good_report}")
if bad_report.ok:
return SelfTest(unit_type, False, f"accepted a broken unit: {bad_report}")
return SelfTest(unit_type, True)