|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +scripts/team_stats.py |
| 4 | +
|
| 5 | +Compute a per-contributor "Translated Count" for TEAM.md by walking every |
| 6 | +.po file and using ``git blame`` to attribute each translated msgstr to the |
| 7 | +author who last changed it. |
| 8 | +
|
| 9 | +Contributors are identified by their git ``user.name``, which is public |
| 10 | +information visible on every commit. Names are mapped to their TEAM.md |
| 11 | +handles through the ``NAME_ALIASES`` table (a git username is not always |
| 12 | +the same as the handle used on GitHub/TEAM.md). |
| 13 | +
|
| 14 | +If two contributors share the same git username, a warning is printed with |
| 15 | +a partially redacted email so the conflict can be resolved by asking one of |
| 16 | +them to update their ``git config user.name``. |
| 17 | +
|
| 18 | +Mechanical commits are excluded: |
| 19 | + * bulk ``Sync translations with CPython`` syncs |
| 20 | + * header-only maintenance commits (``Update .po files``) |
| 21 | + * Transifex/bot import commits and bot accounts |
| 22 | +
|
| 23 | +Requires: pip install polib |
| 24 | +
|
| 25 | +Usage: |
| 26 | + python3 scripts/team_stats.py # report over the whole repo |
| 27 | + python3 scripts/team_stats.py tutorial/ # restrict to a directory |
| 28 | + python3 scripts/team_stats.py --update-teammd # rewrite TEAM.md in-place |
| 29 | +""" |
| 30 | +import argparse |
| 31 | +import re |
| 32 | +import subprocess |
| 33 | +from collections import defaultdict |
| 34 | +from pathlib import Path |
| 35 | + |
| 36 | +import polib |
| 37 | + |
| 38 | +REPO_ROOT = Path(__file__).resolve().parent.parent |
| 39 | + |
| 40 | +BOT_NAME_RE = re.compile( |
| 41 | + r'github[^\w]*actions|\[bot\]|not committed yet', re.IGNORECASE) |
| 42 | +BOT_EMAIL_RE = re.compile( |
| 43 | + r'github-actions|transifex|\[bot\]|@users\.noreply\.github\.com', |
| 44 | + re.IGNORECASE, |
| 45 | +) |
| 46 | +MECHANICAL_SUBJECT_RE = re.compile( |
| 47 | + r'^(?:sync\s+translations\s+with\s+cpython\b' |
| 48 | + r'|update\s+\.po\s+files(?:\s*\(\d+\))?\s*$' |
| 49 | + r'|update\s+farsi\s+translations\s+from\s+transifex\b)', |
| 50 | + re.IGNORECASE, |
| 51 | +) |
| 52 | + |
| 53 | +# git user.name -> TEAM.md User handle (lowercase keys). |
| 54 | +NAME_ALIASES = { |
| 55 | + "sepehr rasouli": "sepehr-rs", |
| 56 | + "revisto": "Revisto", |
| 57 | + "alireza shabani": "Revisto", |
| 58 | + "alireza shabani (revisto)": "Revisto", |
| 59 | + "invincible627": "invincible627", |
| 60 | + "khosro": "khosro_o", |
| 61 | + "ramiz-22": "Ramiz_222", |
| 62 | + "aimer": "aimer.hs872", |
| 63 | +} |
| 64 | + |
| 65 | +# Default role used when a contributor is first added to TEAM.md. |
| 66 | +# Handles that already have a row keep the role stored in TEAM.md; this |
| 67 | +# table is only consulted for brand-new rows. |
| 68 | +NEW_ROW_ROLES = {} |
| 69 | + |
| 70 | +SKIP_DIRS = {".git", ".cpython-src", ".venv", "__pycache__", "venv"} |
| 71 | + |
| 72 | +TEAMMD_ROW_RE = re.compile( |
| 73 | + r'^\|\s*(?P<user>[^|]+?)\s*\|\s*(?P<role>[^|]+?)\s*\|\s*' |
| 74 | + r'(?P<t>\d+(?:\s*\([^)]*\))?)\s*\|\s*(?P<r>\d+)\s*\|\s*(?P<p>\d+)\s*\|$' |
| 75 | +) |
| 76 | + |
| 77 | + |
| 78 | +# --------------------------------------------------------------------------- |
| 79 | +# Privacy helper |
| 80 | +# --------------------------------------------------------------------------- |
| 81 | + |
| 82 | +def redact_email(email: str) -> str: |
| 83 | + """Return a partially redacted email for warning messages. |
| 84 | +
|
| 85 | + ``someone@example.com`` → ``s*****e@e******.com`` |
| 86 | +
|
| 87 | + Only the local part and domain name are obscured; the TLD is kept so |
| 88 | + the domain type is still recognisable. |
| 89 | + """ |
| 90 | + if "@" not in email: |
| 91 | + return "***" |
| 92 | + local, domain = email.rsplit("@", 1) |
| 93 | + |
| 94 | + def _blur(s: str) -> str: |
| 95 | + if len(s) <= 2: |
| 96 | + return s[0] + "*" |
| 97 | + return s[0] + "*" * (len(s) - 2) + s[-1] |
| 98 | + |
| 99 | + if "." in domain: |
| 100 | + domain_name, tld = domain.rsplit(".", 1) |
| 101 | + redacted_domain = f"{_blur(domain_name)}.{tld}" |
| 102 | + else: |
| 103 | + redacted_domain = _blur(domain) |
| 104 | + |
| 105 | + return f"{_blur(local)}@{redacted_domain}" |
| 106 | + |
| 107 | + |
| 108 | +# --------------------------------------------------------------------------- |
| 109 | +# Git helpers |
| 110 | +# --------------------------------------------------------------------------- |
| 111 | + |
| 112 | +def _is_mechanical(subject: str) -> bool: |
| 113 | + return bool(MECHANICAL_SUBJECT_RE.search(subject)) |
| 114 | + |
| 115 | + |
| 116 | +def _is_bot(name: str, email: str) -> bool: |
| 117 | + return bool(BOT_NAME_RE.search(name) or BOT_EMAIL_RE.search(email)) |
| 118 | + |
| 119 | + |
| 120 | +def git_blame_porcelain(path: Path) -> dict[str, dict]: |
| 121 | + """Run ``git blame --porcelain`` and return a commit-hash → info map. |
| 122 | +
|
| 123 | + Each value is a dict with keys: ``name``, ``email``, ``subject``, and |
| 124 | + ``lines`` (a set of 1-based line numbers blamed to that commit). |
| 125 | + """ |
| 126 | + result = subprocess.run( |
| 127 | + ["git", "-C", str(REPO_ROOT), "blame", "--porcelain", |
| 128 | + "--", str(path.resolve().relative_to(REPO_ROOT))], |
| 129 | + capture_output=True, text=True, check=False, |
| 130 | + ) |
| 131 | + if result.returncode != 0: |
| 132 | + return {} |
| 133 | + |
| 134 | + commits: dict[str, dict] = {} |
| 135 | + current_hash = None |
| 136 | + |
| 137 | + for raw_line in result.stdout.splitlines(): |
| 138 | + # Commit header: "<40-char-hash> <orig-line> <result-line> [<num-lines>]" |
| 139 | + parts = raw_line.split() |
| 140 | + if len(parts) >= 3 and len(parts[0]) == 40 and parts[0].isalnum() and parts[1].isdigit() and parts[2].isdigit(): |
| 141 | + h = parts[0] |
| 142 | + result_line = int(parts[2]) |
| 143 | + current_hash = h |
| 144 | + if h not in commits: |
| 145 | + commits[h] = {"name": "", "email": "", "subject": "", "lines": set()} |
| 146 | + commits[h]["lines"].add(result_line) |
| 147 | + elif raw_line.startswith("author ") and current_hash: |
| 148 | + commits[current_hash]["name"] = raw_line[len("author "):].strip() |
| 149 | + elif raw_line.startswith("author-mail ") and current_hash: |
| 150 | + email = raw_line[len("author-mail "):].strip().strip("<>") |
| 151 | + commits[current_hash]["email"] = email.lower() |
| 152 | + elif raw_line.startswith("summary ") and current_hash: |
| 153 | + commits[current_hash]["subject"] = raw_line[len("summary "):] |
| 154 | + |
| 155 | + return commits |
| 156 | + |
| 157 | + |
| 158 | +def check_name_collisions(blame: dict[str, dict]) -> None: |
| 159 | + """Warn if the same git username appears with more than one email address. |
| 160 | +
|
| 161 | + This indicates two distinct people sharing a username, which would cause |
| 162 | + their counts to be merged incorrectly. Ask the affected contributor to |
| 163 | + run ``git config user.name`` to pick a unique name. |
| 164 | + """ |
| 165 | + name_to_emails: dict[str, set[str]] = defaultdict(set) |
| 166 | + for info in blame.values(): |
| 167 | + name = info["name"] |
| 168 | + email = info["email"] |
| 169 | + if name and email and not _is_bot(name, email): |
| 170 | + name_to_emails[name].add(email) |
| 171 | + |
| 172 | + for name, emails in name_to_emails.items(): |
| 173 | + if len(emails) > 1: |
| 174 | + redacted = " vs ".join(redact_email(e) for e in sorted(emails)) |
| 175 | + print( |
| 176 | + f" warning: username '{name}' is used by multiple authors " |
| 177 | + f"({redacted}) — counts may be merged incorrectly. " |
| 178 | + f"Ask one of them to update their git config user.name." |
| 179 | + ) |
| 180 | + |
| 181 | + |
| 182 | +def real_author_for_lines( |
| 183 | + blame: dict[str, dict], |
| 184 | + line_numbers: set[int], |
| 185 | +) -> str | None: |
| 186 | + """Return the git username of the most recent real (non-bot, non-mechanical) |
| 187 | + author who touched any of ``line_numbers``, or None. |
| 188 | +
|
| 189 | + ``git blame --porcelain`` outputs commits in file order, not |
| 190 | + chronologically. We pick the candidate whose blamed lines have the |
| 191 | + highest line number as a proxy for recency, which avoids an extra |
| 192 | + ``git log`` call per entry and is accurate enough for string-level work. |
| 193 | + """ |
| 194 | + best_name: str | None = None |
| 195 | + best_line: int = -1 |
| 196 | + |
| 197 | + for info in blame.values(): |
| 198 | + overlap = info["lines"] & line_numbers |
| 199 | + if not overlap: |
| 200 | + continue |
| 201 | + if _is_bot(info["name"], info["email"]) or _is_mechanical(info["subject"]): |
| 202 | + continue |
| 203 | + candidate_line = max(overlap) |
| 204 | + if candidate_line > best_line: |
| 205 | + best_line = candidate_line |
| 206 | + best_name = info["name"] |
| 207 | + |
| 208 | + return best_name |
| 209 | + |
| 210 | + |
| 211 | +# --------------------------------------------------------------------------- |
| 212 | +# .po file walking |
| 213 | +# --------------------------------------------------------------------------- |
| 214 | + |
| 215 | +def collect_files(paths: list[str]) -> list[Path]: |
| 216 | + files = [] |
| 217 | + for arg in paths: |
| 218 | + p = Path(arg) if Path(arg).is_absolute() else REPO_ROOT / arg |
| 219 | + if p.is_dir(): |
| 220 | + for f in sorted(p.rglob("*.po")): |
| 221 | + rel_parts = set(f.relative_to(REPO_ROOT).parts) |
| 222 | + if rel_parts & SKIP_DIRS: |
| 223 | + continue |
| 224 | + if any(part.startswith(".") for part in f.parts): |
| 225 | + continue |
| 226 | + files.append(f) |
| 227 | + elif p.suffix == ".po": |
| 228 | + files.append(p) |
| 229 | + return files |
| 230 | + |
| 231 | + |
| 232 | +def msgstr_line_numbers(po_path: Path, entry: polib.POEntry) -> set[int]: |
| 233 | + """Return the 1-based line numbers that belong to an entry's msgstr. |
| 234 | +
|
| 235 | + polib exposes ``entry.linenum`` (the msgid line). We scan forward from |
| 236 | + there to find the msgstr block, collecting every continuation line too. |
| 237 | + """ |
| 238 | + lines = po_path.read_text(encoding="utf-8", errors="replace").splitlines() |
| 239 | + total = len(lines) |
| 240 | + i = entry.linenum # 1-based → use as 0-based index (points just past msgid) |
| 241 | + |
| 242 | + # Walk forward to the msgstr line. |
| 243 | + while i < total and not lines[i].startswith("msgstr"): |
| 244 | + i += 1 |
| 245 | + |
| 246 | + # Collect the msgstr line and any quoted continuation lines. |
| 247 | + result: set[int] = set() |
| 248 | + while i < total: |
| 249 | + stripped = lines[i].strip() |
| 250 | + if stripped.startswith("msgstr") or stripped.startswith('"'): |
| 251 | + result.add(i + 1) # convert back to 1-based |
| 252 | + i += 1 |
| 253 | + else: |
| 254 | + break |
| 255 | + |
| 256 | + return result |
| 257 | + |
| 258 | + |
| 259 | +def compute_counts(paths: list[str]) -> dict[str, int]: |
| 260 | + """Walk .po files and accumulate per-contributor translated-entry counts.""" |
| 261 | + counts: dict[str, int] = defaultdict(int) |
| 262 | + |
| 263 | + for po_path in collect_files(paths): |
| 264 | + try: |
| 265 | + po = polib.pofile(str(po_path)) |
| 266 | + except Exception as exc: |
| 267 | + print(f" warning: could not parse {po_path}: {exc}") |
| 268 | + continue |
| 269 | + |
| 270 | + translated = [e for e in po.translated_entries() if "fuzzy" not in e.flags] |
| 271 | + if not translated: |
| 272 | + continue |
| 273 | + |
| 274 | + blame = git_blame_porcelain(po_path) |
| 275 | + if not blame: |
| 276 | + continue |
| 277 | + |
| 278 | + check_name_collisions(blame) |
| 279 | + |
| 280 | + for entry in translated: |
| 281 | + line_nums = msgstr_line_numbers(po_path, entry) |
| 282 | + if not line_nums: |
| 283 | + continue |
| 284 | + name = real_author_for_lines(blame, line_nums) |
| 285 | + if name: |
| 286 | + counts[NAME_ALIASES.get(name.lower(), name)] += 1 |
| 287 | + else: |
| 288 | + counts["(unassigned)"] += 1 |
| 289 | + |
| 290 | + return dict(counts) |
| 291 | + |
| 292 | + |
| 293 | +# --------------------------------------------------------------------------- |
| 294 | +# Output |
| 295 | +# --------------------------------------------------------------------------- |
| 296 | + |
| 297 | +def print_report(counts: dict[str, int]) -> None: |
| 298 | + total = sum(counts.values()) |
| 299 | + print(f"Total non-fuzzy translated entries attributed: {total}\n") |
| 300 | + print("| Contributor | Translated |") |
| 301 | + print("|:------------|-----------:|") |
| 302 | + for user, count in sorted(counts.items(), key=lambda kv: -kv[1]): |
| 303 | + print(f"| {user} | {count} |") |
| 304 | + |
| 305 | + |
| 306 | +def update_teammd(counts: dict[str, int]) -> None: |
| 307 | + """Rewrite the Translated column in TEAM.md; append rows for new handles.""" |
| 308 | + path = REPO_ROOT / "TEAM.md" |
| 309 | + lines = path.read_text(encoding="utf-8").splitlines() |
| 310 | + |
| 311 | + # Build an index of existing rows by username. |
| 312 | + rows: dict[str, tuple[int, str, str, str]] = {} |
| 313 | + for i, line in enumerate(lines): |
| 314 | + m = TEAMMD_ROW_RE.match(line) |
| 315 | + if m: |
| 316 | + rows[m.group("user")] = (i, m.group("role"), m.group("r"), m.group("p")) |
| 317 | + |
| 318 | + touched: list[str] = [] |
| 319 | + new_rows: list[str] = [] |
| 320 | + |
| 321 | + for user, count in sorted(counts.items(), key=lambda kv: -kv[1]): |
| 322 | + if user == "(unassigned)" or count <= 0: |
| 323 | + continue |
| 324 | + if user in rows: |
| 325 | + i, role, r, p = rows[user] |
| 326 | + lines[i] = f"| {user} | {role} | {count} | {r} | {p} |" |
| 327 | + touched.append(user) |
| 328 | + else: |
| 329 | + role = NEW_ROW_ROLES.get(user, "translator") |
| 330 | + new_rows.append(f"| {user} | {role} | {count} | 0 | 0 |") |
| 331 | + touched.append(user) |
| 332 | + |
| 333 | + if new_rows: |
| 334 | + lines.extend(new_rows) |
| 335 | + |
| 336 | + path.write_text("\n".join(lines) + "\n", encoding="utf-8") |
| 337 | + print(f"TEAM.md: updated {len(touched)} contributor row(s)") |
| 338 | + if new_rows: |
| 339 | + print(f" added {len(new_rows)} new row(s) — review roles manually") |
| 340 | + |
| 341 | + |
| 342 | +# --------------------------------------------------------------------------- |
| 343 | +# Entry point |
| 344 | +# --------------------------------------------------------------------------- |
| 345 | + |
| 346 | +def main() -> None: |
| 347 | + parser = argparse.ArgumentParser( |
| 348 | + description=__doc__, |
| 349 | + formatter_class=argparse.RawDescriptionHelpFormatter, |
| 350 | + ) |
| 351 | + parser.add_argument( |
| 352 | + "paths", nargs="*", default=["."], |
| 353 | + help="Files or directories to scan (default: whole repo)", |
| 354 | + ) |
| 355 | + parser.add_argument( |
| 356 | + "--update-teammd", action="store_true", |
| 357 | + help="rewrite TEAM.md from the computed counts", |
| 358 | + ) |
| 359 | + args = parser.parse_args() |
| 360 | + |
| 361 | + counts = compute_counts(args.paths) |
| 362 | + |
| 363 | + if args.update_teammd: |
| 364 | + update_teammd(counts) |
| 365 | + else: |
| 366 | + print_report(counts) |
| 367 | + |
| 368 | + |
| 369 | +if __name__ == "__main__": |
| 370 | + main() |
0 commit comments