|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +"""Known security advisories for a set of pinned packages, from osv.dev. |
| 3 | +
|
| 4 | +One batch query covers the whole set; details are fetched only for the hits. |
| 5 | +Needs network access, no key. |
| 6 | +""" |
| 7 | +import json |
| 8 | +import re |
| 9 | +import urllib.request |
| 10 | +from concurrent.futures import ThreadPoolExecutor |
| 11 | +from pathlib import Path |
| 12 | + |
| 13 | +try: |
| 14 | + from packaging.version import Version |
| 15 | +except ImportError: |
| 16 | + from pip._vendor.packaging.version import Version |
| 17 | + |
| 18 | +OSV = "https://api.osv.dev/v1" |
| 19 | +TIMEOUT = 30 |
| 20 | +BATCH = 1000 # osv.dev limit per querybatch |
| 21 | + |
| 22 | + |
| 23 | +def _call(url, payload=None): |
| 24 | + data = json.dumps(payload).encode() if payload is not None else None |
| 25 | + request = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}) |
| 26 | + with urllib.request.urlopen(request, timeout=TIMEOUT) as response: |
| 27 | + return json.load(response) |
| 28 | + |
| 29 | + |
| 30 | +def pins_from_file(path): |
| 31 | + """(name, version) pairs of a pylock.toml or of the `name==version` lines of a requirements file.""" |
| 32 | + path = Path(path) |
| 33 | + text = path.read_text(encoding="utf-8") |
| 34 | + if path.suffix == ".toml": |
| 35 | + try: |
| 36 | + import tomllib |
| 37 | + except ImportError: |
| 38 | + import tomli as tomllib |
| 39 | + return [(p["name"], p["version"]) for p in tomllib.loads(text).get("packages", []) if p.get("version")] |
| 40 | + pins = [] |
| 41 | + for line in text.splitlines(): |
| 42 | + match = re.match(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?\s*===?\s*([^\s;#\\]+)", line) |
| 43 | + if match: |
| 44 | + pins.append(match.groups()) |
| 45 | + return pins |
| 46 | + |
| 47 | + |
| 48 | +def _fixed_in(vuln, name, installed): |
| 49 | + """Lowest version that fixes `vuln` above `installed`, '' when none is known.""" |
| 50 | + normal = re.sub(r"[-_.]+", "-", name).lower() |
| 51 | + fixes = [] |
| 52 | + for affected in vuln.get("affected", []): |
| 53 | + package = affected.get("package", {}) |
| 54 | + if package.get("ecosystem") != "PyPI" or re.sub(r"[-_.]+", "-", package.get("name", "")).lower() != normal: |
| 55 | + continue |
| 56 | + for span in affected.get("ranges", []): |
| 57 | + for event in span.get("events", []): |
| 58 | + try: |
| 59 | + if "fixed" in event and Version(event["fixed"]) > Version(installed): |
| 60 | + fixes.append(Version(event["fixed"])) |
| 61 | + except Exception: |
| 62 | + pass |
| 63 | + return str(min(fixes)) if fixes else "" |
| 64 | + |
| 65 | + |
| 66 | +def audit(pins): |
| 67 | + """Advisories per pinned package: [{package, version, id, aliases, fixed, summary}], sorted. |
| 68 | +
|
| 69 | + The same flaw is often published twice (PYSEC and GHSA, aliases of each |
| 70 | + other): it is reported once. A PYSEC record whose GHSA alias does not match |
| 71 | + the same version is dropped: PYSEC often marks every version affected when |
| 72 | + only a git commit is known as the fix, GHSA carries the curated range. |
| 73 | + """ |
| 74 | + pins = sorted(set(pins), key=lambda p: p[0].lower()) |
| 75 | + hits = [] |
| 76 | + for start in range(0, len(pins), BATCH): |
| 77 | + chunk = pins[start:start + BATCH] |
| 78 | + queries = [{"package": {"name": n, "ecosystem": "PyPI"}, "version": v} for n, v in chunk] |
| 79 | + answers = _call(f"{OSV}/querybatch", {"queries": queries}).get("results", []) |
| 80 | + for (name, version), answer in zip(chunk, answers): |
| 81 | + hits += [(name, version, v["id"]) for v in answer.get("vulns") or []] |
| 82 | + with ThreadPoolExecutor(8) as pool: |
| 83 | + details = dict(zip((h[2] for h in hits), pool.map(lambda h: _call(f"{OSV}/vulns/{h[2]}"), hits))) |
| 84 | + rows, seen = [], set() |
| 85 | + for name, version, vid in sorted(hits, key=lambda h: (h[0].lower(), not h[2].startswith("GHSA"), h[2])): |
| 86 | + vuln = details[vid] |
| 87 | + aliases = [a for a in vuln.get("aliases", []) if a != vid] |
| 88 | + if (name, vid) in seen: |
| 89 | + continue |
| 90 | + if vid.startswith("PYSEC") and any(a.startswith("GHSA") for a in aliases): |
| 91 | + continue # its GHSA alias was not a hit, or it would be in seen |
| 92 | + seen.update((name, a) for a in [vid] + aliases) |
| 93 | + rows.append({"package": name, "version": version, "id": vid, "aliases": aliases, |
| 94 | + "fixed": _fixed_in(vuln, name, version), |
| 95 | + "summary": vuln.get("summary") or (vuln.get("details") or "").split("\n")[0]}) |
| 96 | + return rows |
| 97 | + |
| 98 | + |
| 99 | +def upgrades(rows): |
| 100 | + """Per affected package, the version fixing all its advisories, and whether it is clean. |
| 101 | +
|
| 102 | + The highest `fixed` can carry an advisory of its own (pypdf needed eleven |
| 103 | + releases to get past that), so the suggested versions are audited too: one |
| 104 | + more batch query. Not clean means a later release is needed; it is not |
| 105 | + searched for. {package: {"version", "to", "clean"}}, `to` '' and `clean` |
| 106 | + None when some advisory has no known fix. |
| 107 | + """ |
| 108 | + result = {} |
| 109 | + for row in rows: |
| 110 | + entry = result.setdefault(row["package"], {"version": row["version"], "fixes": []}) |
| 111 | + entry["fixes"].append(row["fixed"]) |
| 112 | + for entry in result.values(): |
| 113 | + fixes = entry.pop("fixes") |
| 114 | + entry["to"] = max(fixes, key=Version) if all(fixes) else "" |
| 115 | + entry["clean"] = None |
| 116 | + proposed = [(name, entry["to"]) for name, entry in result.items() if entry["to"]] |
| 117 | + still = {row["package"] for row in audit(proposed)} if proposed else set() |
| 118 | + for name, entry in result.items(): |
| 119 | + if entry["to"]: |
| 120 | + entry["clean"] = name not in still |
| 121 | + return result |
0 commit comments