Repository navigation
Expand file tree
/
Copy pathdiff_extractor.py
More file actions
408 lines (355 loc) · 14.5 KB
/
Copy pathdiff_extractor.py
File metadata and controls
408 lines (355 loc) · 14.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
"""Extract structured diffs from git via subprocess; no GitPython, no raw strings."""
from __future__ import annotations
import os
import subprocess
from collections.abc import Iterable
from dataclasses import dataclass
from fnmatch import fnmatchcase
from pathlib import Path, PurePosixPath
from typing import Literal, Protocol
FileStatus = Literal["added", "modified", "deleted", "renamed"]
#: Built-in noise filter; users extend (never shrink) it via `ignore_globs`.
DEFAULT_IGNORE_GLOBS: tuple[str, ...] = (
"package-lock.json",
"poetry.lock",
"Cargo.lock",
"*.min.js",
"*.bundle.js",
"*.map",
"dist/*",
"*.snap",
"*.lock",
)
class GitCommandError(Exception):
"""Raised when a git invocation fails for any reason."""
class NotAGitRepo(GitCommandError):
"""Raised when the target directory is not inside a git repository."""
@dataclass
class FileDiff:
"""One file's diff; every downstream layer works off this, not raw text."""
path: str
old_path: str | None
status: FileStatus
additions: int
deletions: int
patch_text: str
is_binary: bool
class GitRunner(Protocol):
"""Runs git so tests can substitute a fake; production shells out for real."""
def run(self, args: list[str], cwd: Path) -> str:
"""Run `git <args>` in cwd, returning stdout or raising GitCommandError."""
...
class SubprocessGitRunner:
"""GitRunner that calls the real git binary with a timeout."""
def __init__(self, timeout: int = 30) -> None:
self._timeout = timeout
def run(self, args: list[str], cwd: Path) -> str:
"""Run the real git binary; map failures to typed exceptions."""
if not cwd.exists():
raise NotAGitRepo(f"{cwd} does not exist.")
try:
proc = subprocess.run(
["git", *args],
cwd=cwd,
capture_output=True,
text=True,
timeout=self._timeout,
check=False,
# Deterministic English errors regardless of host locale.
env={**os.environ, "LC_ALL": "C"},
)
except FileNotFoundError as exc:
raise GitCommandError("git binary not found on PATH.") from exc
except subprocess.TimeoutExpired as exc:
raise GitCommandError(f"git {' '.join(args)} timed out.") from exc
except UnicodeDecodeError as exc:
raise GitCommandError(f"git {' '.join(args)} output was not valid UTF-8.") from exc
if proc.returncode != 0:
err = proc.stderr.strip()
if "not a git repository" in err:
raise NotAGitRepo(f"{cwd} is not inside a git repository.")
raise GitCommandError(f"git {' '.join(args)} failed: {err}")
return proc.stdout
def _dir_match(posix: str, dirname: str) -> bool:
"""True when dirname is the path itself or any parent directory in it."""
return posix == dirname or posix.startswith(dirname + "/") or f"/{dirname}/" in f"/{posix}"
def matches_any(path: str, patterns: Iterable[str]) -> bool:
"""Gitignore-flavored match: dir prefixes, bare basenames, else fnmatch.
Directory patterns (`dist/`, `dist/*`) match at any depth, so monorepo
`packages/ui/dist/` output is filtered just like root-level `dist/`.
"""
posix = PurePosixPath(path).as_posix()
name = PurePosixPath(path).name
for pat in patterns:
if pat.endswith("/*"):
if _dir_match(posix, pat[:-2]):
return True
elif pat.endswith("/"):
if _dir_match(posix, pat[:-1]):
return True
elif "/" not in pat:
if fnmatchcase(name, pat):
return True
elif fnmatchcase(posix, pat):
return True
return False
_STATUS_MAP: dict[str, FileStatus] = {
"A": "added",
"M": "modified",
"D": "deleted",
"R": "renamed",
"T": "modified",
"C": "added",
"U": "modified",
}
def _parse_numstat(output: str) -> tuple[dict[str, tuple[int, int, str | None]], set[str]]:
"""Parse `git diff -M -z --numstat`, keyed by new path with binary set.
With -z, records are NUL-separated and a rename carries no inline path at
all: the `added\\tdeleted\\t` prefix is followed by an empty field, then
the old and new paths as two further NUL-terminated fields. This avoids
the `{old => new}` pretty-printing entirely, so no arrow-parsing is needed.
"""
counts: dict[str, tuple[int, int, str | None]] = {}
binary: set[str] = set()
fields = output.split("\0")
i = 0
while i < len(fields):
field = fields[i]
i += 1
if not field:
continue
parts = field.split("\t")
if len(parts) < 3:
raise GitCommandError(f"cannot parse numstat record: {field!r}.")
added_s, deleted_s, rest = parts[0], parts[1], "\t".join(parts[2:])
old_path: str | None
if rest == "":
# Rename record: old and new paths follow as two extra fields.
if i + 1 >= len(fields) or not fields[i] or not fields[i + 1]:
raise GitCommandError(f"cannot parse numstat rename: {field!r}.")
old, key = fields[i], fields[i + 1]
old_path = old
i += 2
else:
key, old_path = rest, None
if added_s == "-" or deleted_s == "-":
counts[key] = (0, 0, old_path)
binary.add(key)
continue
try:
added, deleted = int(added_s), int(deleted_s)
except ValueError as exc:
raise GitCommandError(f"cannot parse numstat counts: {field!r}.") from exc
counts[key] = (added, deleted, old_path)
return counts, binary
def _parse_name_status(output: str) -> dict[str, tuple[FileStatus, str | None]]:
"""Parse `git diff -M --name-status`; clean `R100\\told\\tnew` rename rows."""
statuses: dict[str, tuple[FileStatus, str | None]] = {}
for line in output.splitlines():
if not line:
continue
parts = line.split("\t")
code = parts[0][0]
if code not in _STATUS_MAP:
raise GitCommandError(f"unknown git status code: {line!r}.")
status = _STATUS_MAP[code]
if code in ("R", "C"):
if len(parts) != 3:
raise GitCommandError(f"cannot parse rename/copy row: {line!r}.")
_, old, new = parts
statuses[_decode_quoted(new)] = (status, _decode_quoted(old))
else:
if len(parts) != 2:
raise GitCommandError(f"cannot parse status row: {line!r}.")
statuses[_decode_quoted(parts[1])] = (status, None)
return statuses
def _split_header_fallback(rest: str) -> tuple[str | None, str | None]:
"""Split `a/X b/Y` across quoted/unquoted combinations on either side.
Our git flags pin the prefixes, so Y always begins `b/` or `"b/`, and
rfind lets the true boundary win over interior lookalikes. Rename/copy
metadata and ---/+++ re-keying backstop anything stranger still.
"""
if rest.endswith('"'):
idx = rest.rfind(' "b/')
if idx == -1:
return None, None
return _strip_git_prefix(rest[:idx]), _strip_git_prefix(rest[idx + 1 :])
idx = rest.rfind(" b/")
if idx < 2:
return None, None
return _strip_git_prefix(rest[:idx]), _strip_git_prefix(rest[idx + 1 :])
def _unescape_c_quoting(text: str) -> str:
"""Decode git's C-style path escapes (`\\\\`, `\\"`, `\\n`, `\\t`, octal).
Only meaningful on paths git quoted; unquoted output (e.g. `-z --numstat`)
is never munged and must pass through untouched. Octal runs decode as
UTF-8 bytes because git escapes non-ASCII per byte, not per character.
"""
out: list[str] = []
raw = bytearray()
def flush_octal() -> None:
if raw:
out.append(bytes(raw).decode("utf-8", errors="surrogateescape"))
raw.clear()
i, n = 0, len(text)
while i < n:
ch = text[i]
if ch != "\\" or i + 1 >= n:
flush_octal()
out.append(ch)
i += 1
continue
nxt = text[i + 1]
if nxt in "01234567":
digits = text[i + 1 : i + 4]
length = 0
while length < 3 and length < len(digits) and digits[length] in "01234567":
length += 1
raw.extend(int(digits[:length], 8).to_bytes(1, "big"))
i += 1 + length
continue
flush_octal()
if nxt == "n":
out.append("\n")
elif nxt == "t":
out.append("\t")
elif nxt == "\\":
out.append("\\")
elif nxt == '"':
out.append('"')
else:
out.append(ch)
out.append(nxt)
i += 2
flush_octal()
return "".join(out)
def _decode_quoted(path: str) -> str:
"""Remove git's C-style quoting when present; bare paths pass through."""
if len(path) >= 2 and path.startswith('"') and path.endswith('"'):
return _unescape_c_quoting(path[1:-1])
return path
def _strip_git_prefix(path: str) -> str:
"""Strip git's a//b/ prefixes, unescaping C-style quoting when present."""
path = _decode_quoted(path)
if path.startswith(("a/", "b/")):
return path[2:]
return path
def _parse_patch(output: str) -> dict[str, str]:
"""Split `--patch` output per file, keyed by new path (old on deletion)."""
chunks: dict[str, list[str]] = {}
order: list[str] = []
current: str | None = None
minus: str | None = None
chunk: list[str] = []
def flush() -> None:
nonlocal current, minus
if current is not None or minus is not None:
# Rename/copy lines carry bare paths (never a//b/ prefixed), so
# decode quoting only; stripping would eat real a//b/ directories.
for cl in chunk:
if cl.startswith(("rename to ", "copy to ")):
current = _decode_quoted(cl.split(" ", 2)[2].rstrip("\n"))
elif cl.startswith(("rename from ", "copy from ")):
minus = _decode_quoted(cl.split(" ", 2)[2].rstrip("\n"))
key = current if current is not None and current != "/dev/null" else minus
if key is not None:
if key not in chunks:
chunks[key] = []
order.append(key)
chunks[key].extend(chunk)
for line in output.splitlines(keepends=True):
if line.startswith("diff --git "):
flush()
rest = line[len("diff --git ") :].rstrip("\n")
if len(rest) >= 5 and rest.startswith("a/"):
mid = (len(rest) - 1) // 2
if rest[mid : mid + 3] == " b/" and rest[2:mid] == rest[mid + 3 :]:
minus = _strip_git_prefix(rest[2:mid])
current = _strip_git_prefix(rest[mid + 3 :])
else:
minus, current = _split_header_fallback(rest)
else:
minus, current = _split_header_fallback(rest)
chunk = [line]
elif current is None and minus is None:
continue
else:
chunk.append(line)
flush()
# Re-key via the ---/+++ lines: +++ names the new file (/dev/null on delete).
fixed: dict[str, str] = {}
for key in order:
text = "".join(chunks[key])
for text_line in text.splitlines():
if text_line.startswith("+++ "):
plus = _strip_git_prefix(text_line[4:].strip())
if plus == "/dev/null":
for prev in text.splitlines():
if prev.startswith("--- "):
key = _strip_git_prefix(prev[4:].strip())
break
else:
key = plus
break
fixed[key] = text
return fixed
class DiffExtractor:
"""Three git-diff modes sharing one parse/merge/filter pipeline."""
def __init__(
self,
repo: Path | None = None,
runner: GitRunner | None = None,
extra_ignores: Iterable[str] = (),
) -> None:
self._repo = repo if repo is not None else Path.cwd()
self._runner = runner if runner is not None else SubprocessGitRunner()
self._ignores = (*DEFAULT_IGNORE_GLOBS, *extra_ignores)
def staged(self) -> list[FileDiff]:
"""Diff of staged changes (the `git commit` path)."""
return self._extract(["diff", "--staged"])
def branch_comparison(self, base: str) -> list[FileDiff]:
"""Three-dot `base...HEAD` diff (the PR-summary path)."""
return self._extract(["diff", f"{base}...HEAD"])
def rev_range(self, from_ref: str, to_ref: str) -> list[FileDiff]:
"""Two-dot `from..to` diff (the changelog path)."""
return self._extract(["diff", f"{from_ref}..{to_ref}"])
def _extract(self, base_args: list[str]) -> list[FileDiff]:
"""Run the three git calls and merge them keyed by path, never by order."""
self._runner.run(["rev-parse", "--git-dir"], self._repo)
git_flags = [
"-c",
"core.quotepath=false",
"-c",
"diff.mnemonicPrefix=false",
*base_args,
"--src-prefix=a/",
"--dst-prefix=b/",
"--no-color",
"--no-ext-diff",
"--no-textconv",
]
numstat = self._runner.run([*git_flags, "-M", "-z", "--numstat"], self._repo)
patch = self._runner.run([*git_flags, "-M", "--patch"], self._repo)
names = self._runner.run([*git_flags, "-M", "--name-status"], self._repo)
counts, binary = _parse_numstat(numstat)
statuses = _parse_name_status(names)
patches = _parse_patch(patch)
diffs: list[FileDiff] = []
for path in list(counts) + [p for p in statuses if p not in counts]:
if matches_any(path, self._ignores):
continue
added, deleted, num_old = counts.get(path, (0, 0, None))
status, name_old = statuses.get(path, ("modified", None))
old_path = name_old if name_old is not None else num_old
is_binary = path in binary
diffs.append(
FileDiff(
path=path,
old_path=old_path,
status=status,
additions=added,
deletions=deleted,
patch_text="" if is_binary else patches.get(path, ""),
is_binary=is_binary,
)
)
return diffs