-
Notifications
You must be signed in to change notification settings - Fork 18
Expand file tree
/
Copy pathopenapi_spec.py
More file actions
213 lines (153 loc) · 8.45 KB
/
Copy pathopenapi_spec.py
File metadata and controls
213 lines (153 loc) · 8.45 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
"""Fetch the published OpenAPI specification and record the version the models were generated from.
`fetch` downloads the specification into git-ignored `tmp/`, and both codegen passes read that one copy, so a
specification redeployed mid-run can't yield models built from two different inputs. Only the *version* is
committed, in `[tool.apify.openapi-spec]` in `pyproject.toml`.
`record-version` writes it last, once generation succeeded, so it names the specification the committed
`_models.py`, `_typeddicts.py`, and `_literals.py` follow from. `recorded-version` prints it; the nightly workflow
reads it before regenerating to report whether the stamp moved.
The stamp is a coarse marker, not a content identity: the specification is served latest-only, so it can't be
fetched back, and apify-docs bumps it in a follow-up `[skip ci]` commit, so a deploy can publish new content under
the old stamp.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
import time
import tomllib
from pathlib import Path
from typing import NoReturn
import impit
REPO_ROOT = Path(__file__).resolve().parent.parent
# The published, bundled specification, built and deployed from the `apify/apify-docs` repository.
SPEC_URL = 'https://docs.apify.com/api/openapi.json'
# Codegen input, deliberately outside version control - `tmp/` is git-ignored.
SPEC_PATH = REPO_ROOT / 'tmp' / 'openapi.json'
PYPROJECT_PATH = REPO_ROOT / 'pyproject.toml'
VERSION_TABLE_PATH = ('tool', 'apify', 'openapi-spec')
VERSION_TABLE = f'[{".".join(VERSION_TABLE_PATH)}]'
VERSION_KEY = 'version'
# Writing replaces the value in place to keep the surrounding comments. The patterns tolerate the whitespace and
# trailing comments TOML allows, so reformatting `pyproject.toml` can't quietly break the nightly regeneration.
TABLE_HEADER_PATTERN = re.compile(rf'^\s*\[\s*{re.escape(".".join(VERSION_TABLE_PATH))}\s*\]\s*(?:#.*)?$')
ANY_TABLE_HEADER_PATTERN = re.compile(r'^\s*\[')
VERSION_ENTRY_PATTERN = re.compile(rf'^(?P<prefix>\s*{VERSION_KEY}\s*=\s*)"(?P<value>[^"]*)"(?P<suffix>.*)$')
# An error page or a truncated response must never be generated from; the real specification is roughly 1 MB.
MIN_SPEC_SIZE_BYTES = 100_000
# Members every specification we can generate models from has to contain; `info` carries the recorded version.
REQUIRED_SPEC_KEYS = ('openapi', 'info', 'paths', 'components')
REQUEST_TIMEOUT_SECS = 60
# The nightly workflow alerts the team when this fails, so a single network blip shouldn't be worth a ping.
DOWNLOAD_ATTEMPTS = 3
RETRY_DELAY_SECS = 5
def fail(message: str) -> NoReturn:
"""Report a failure on stderr and exit non-zero."""
print(message, file=sys.stderr)
sys.exit(1)
def download_spec() -> bytes:
"""Download the published specification, retrying transient failures."""
last_error = ''
with impit.Client(follow_redirects=True) as client:
for attempt in range(1, DOWNLOAD_ATTEMPTS + 1):
try:
response = client.request('GET', SPEC_URL, timeout=REQUEST_TIMEOUT_SECS)
except Exception as exc:
last_error = f'{type(exc).__name__}: {exc}'
else:
if response.status_code == 200:
return response.content
last_error = f'HTTP {response.status_code}'
print(f'Attempt {attempt}/{DOWNLOAD_ATTEMPTS} to download {SPEC_URL} failed ({last_error}).')
if attempt < DOWNLOAD_ATTEMPTS:
time.sleep(RETRY_DELAY_SECS)
fail(f'Failed to download {SPEC_URL} after {DOWNLOAD_ATTEMPTS} attempts: {last_error}.')
def read_spec_version(payload: bytes) -> str:
"""Validate a downloaded specification and return its `info.version`."""
if len(payload) < MIN_SPEC_SIZE_BYTES:
fail(f'Downloaded specification is only {len(payload)} bytes, which cannot be the real one - aborting.')
try:
spec = json.loads(payload)
except json.JSONDecodeError as exc:
fail(f'Downloaded specification is not valid JSON: {exc}.')
if not isinstance(spec, dict):
fail(f'Downloaded specification is a JSON {type(spec).__name__}, not an object - aborting.')
missing_keys = [key for key in REQUIRED_SPEC_KEYS if key not in spec]
if missing_keys:
fail(f'Downloaded specification is missing top-level {", ".join(missing_keys)} - aborting.')
info = spec['info']
version = info.get(VERSION_KEY) if isinstance(info, dict) else None
if not isinstance(version, str) or not version:
fail('Downloaded specification has no `info.version` string - aborting.')
return version
def fetch() -> None:
"""Download the published specification for codegen to read."""
payload = download_spec()
version = read_spec_version(payload)
# Written byte for byte, so the key order the generator sees is the published one: `keep_model_order` ties
# the order of the generated models to it.
SPEC_PATH.parent.mkdir(parents=True, exist_ok=True)
SPEC_PATH.write_bytes(payload)
print(f'Wrote {SPEC_PATH.relative_to(REPO_ROOT)} (version {version}, {len(payload)} bytes).')
def read_recorded_version() -> str:
"""Return the specification version currently recorded in `pyproject.toml`."""
try:
config = tomllib.loads(PYPROJECT_PATH.read_text(encoding='utf-8'))
except tomllib.TOMLDecodeError as exc:
fail(f'{PYPROJECT_PATH.name} is not valid TOML: {exc}.')
for key in VERSION_TABLE_PATH:
if not isinstance(config, dict) or key not in config:
fail(f'{PYPROJECT_PATH.name} has no {VERSION_TABLE} table - cannot read the specification version.')
config = config[key]
version = config.get(VERSION_KEY) if isinstance(config, dict) else None
if not isinstance(version, str) or not version:
fail(f'{VERSION_TABLE} in {PYPROJECT_PATH.name} has no `{VERSION_KEY}` string.')
return version
def write_recorded_version(version: str) -> None:
"""Replace the recorded specification version in `pyproject.toml`, leaving the rest of the file untouched."""
lines = PYPROJECT_PATH.read_text(encoding='utf-8').splitlines(keepends=True)
try:
table_index = next(index for index, line in enumerate(lines) if TABLE_HEADER_PATTERN.match(line.rstrip('\n')))
except StopIteration:
fail(f'{PYPROJECT_PATH.name} has no {VERSION_TABLE} table - cannot record the specification version.')
# Only the table's own entries may be rewritten, so a missing key can't silently hit the next table's `version`.
for index in range(table_index + 1, len(lines)):
line = lines[index].rstrip('\n')
if ANY_TABLE_HEADER_PATTERN.match(line):
break
match = VERSION_ENTRY_PATTERN.match(line)
if match:
lines[index] = f'{match["prefix"]}"{version}"{match["suffix"]}\n'
PYPROJECT_PATH.write_text(''.join(lines), encoding='utf-8', newline='\n')
return
fail(f'{VERSION_TABLE} in {PYPROJECT_PATH.name} has no `{VERSION_KEY}` entry - cannot record the version.')
def record_version() -> None:
"""Record the fetched specification's version in `pyproject.toml`."""
if not SPEC_PATH.is_file():
fail(f'{SPEC_PATH.relative_to(REPO_ROOT)} is missing - run `poe generate-models` instead of this alone.')
version = read_spec_version(SPEC_PATH.read_bytes())
previous = read_recorded_version()
if previous == version:
print(f'Specification version {version} is already recorded in {PYPROJECT_PATH.name}.')
return
write_recorded_version(version)
print(f'Recorded specification version in {PYPROJECT_PATH.name}: {previous} -> {version}.')
def recorded_version() -> None:
"""Print the recorded specification version, for the regeneration workflow to read."""
print(read_recorded_version())
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(required=True)
fetch_parser = subparsers.add_parser('fetch', help='download the published specification into tmp/')
fetch_parser.set_defaults(handler=fetch)
record_parser = subparsers.add_parser(
'record-version', help="record the fetched specification's version in pyproject.toml"
)
record_parser.set_defaults(handler=record_version)
show_parser = subparsers.add_parser(
'recorded-version', help='print the specification version recorded in pyproject.toml'
)
show_parser.set_defaults(handler=recorded_version)
parser.parse_args().handler()
if __name__ == '__main__':
main()