melonakos · Aug 2, 2025 · Jul 12, 2026 · Oct 24, 2026 · Oct 24, 2026 · Oct 24, 2026
diff --git a/.github/fixtures/README.md b/.github/fixtures/README.md
@@ -0,0 +1,16 @@
+# Matrix fixtures for CI
+
+`matrixmarket.tar.gz` contains the Matrix Market fixtures listed in
+`test/CMakeModules/download_sparse_datasets.cmake`. They are public scientific
+datasets from the [SuiteSparse Matrix Collection](https://sparse.tamu.edu/).
+The original matrix headers, author credits, and collection metadata remain
+in each file.
+
+The bundle comes from an existing ArrayFire download cache. Each file matches
+its retained upstream archive. `matrixmarket.json` records the source URLs,
+archive and file SHA-256 checksums, dimensions, and entry counts. CI verifies
+the bundle before restoring it and also reuses the GitHub Actions cache.
+
+Keep the fixture list, bundle, and manifest together when changing these tests.
+The bundle lets the existing tests run during collection outages without
+repeated downloads from the provider.
diff --git a/.github/fixtures/matrixmarket.json b/.github/fixtures/matrixmarket.json
@@ -0,0 +1,87 @@
+{
+  "source": "https://sparse.tamu.edu/",
+  "archive_sha256": "a6806c0e5f47be63f60713686382414f372bc98afe7410cbafe72362fcc22c84",
+  "files": {
+    "JGD_Kocay/Trec4/Trec4.mtx": {
+      "sha256": "c33e82bc1728021c4a29937f502a610dda9c7a81748bff47badc65e4bf3b7408",
+      "bytes": 2263,
+      "rows": 2,
+      "columns": 3,
+      "entries": 3,
+      "download_url": "https://sparse.tamu.edu/MM/JGD_Kocay/Trec4.tar.gz",
+      "original_archive_sha256": "f49eedd1c2904df742cd492e0e30fc928bcf904f0ec1c4225a05c9e8d8f1e30d"
+    },
+    "HB/bcsstm02/bcsstm02.mtx": {
+      "sha256": "c8c629d8ea30269b3ee71a0fce17914e463359e817ae4553d3745b98f0cddae1",
+      "bytes": 1884,
+      "rows": 66,
+      "columns": 66,
+      "entries": 66,
+      "download_url": "https://sparse.tamu.edu/MM/HB/bcsstm02.tar.gz",
+      "original_archive_sha256": "72897c2859efed23dad58748683dc094fbbf2ea63532f83b8deedbcb961c7c98"
+    },
+    "HB/young4c/young4c.mtx": {
+      "sha256": "06db1d6ff551065237809e3d79991e1788ab24eae295a1cd3eb37dab342a59e5",
+      "bytes": 61922,
+      "rows": 841,
+      "columns": 841,
+      "entries": 4089,
+      "download_url": "https://sparse.tamu.edu/MM/HB/young4c.tar.gz",
+      "original_archive_sha256": "1f53a17ddfc4fdfd291db74ef02eb8c3edfc65b799e62a9feb3794fe42e05c3e"
+    },
+    "LPnetlib/lpi_vol1/lpi_vol1.mtx": {
+      "sha256": "325ad6f923f11926add3481621f90eb08003377f69377756477fc05067bac3e0",
+      "bytes": 23421,
+      "rows": 323,
+      "columns": 464,
+      "entries": 1646,
+      "download_url": "https://sparse.tamu.edu/MM/LPnetlib/lpi_vol1.tar.gz",
+      "original_archive_sha256": "0e9a9897560ad0829030dad1adea06cd4fb0dd50fcc3910132d9bc00f1db6bc9"
+    },
+    "LPnetlib/lpi_qual/lpi_qual.mtx": {
+      "sha256": "195e2e19664ebf88fb0eeb25e882934fb05e176d7c0b867752a7610caa57dbde",
+      "bytes": 23342,
+      "rows": 323,
+      "columns": 464,
+      "entries": 1646,
+      "download_url": "https://sparse.tamu.edu/MM/LPnetlib/lpi_qual.tar.gz",
+      "original_archive_sha256": "54fb7b137891c1d82fe349854d0a7719cfe1e8dab1e23ab3e4ec230a29341873"
+    },
+    "Sandia/oscil_dcop_12/oscil_dcop_12.mtx": {
+      "sha256": "8da3dcf47cc90668c9d808c389b261e0ef32cb6411095810f96a71cde9161155",
+      "bytes": 31102,
+      "rows": 430,
+      "columns": 430,
+      "entries": 1544,
+      "download_url": "https://sparse.tamu.edu/MM/Sandia/oscil_dcop_12.tar.gz",
+      "original_archive_sha256": "fad82385c1d134640e581f698eac6e835a2d9d9a9b7d6db9892d4b9919630da7"
+    },
+    "Sandia/oscil_dcop_42/oscil_dcop_42.mtx": {
+      "sha256": "a1cfe04f2891763afea28064ca8c1f3337e86aac652fdcdf86afe81577966cbf",
+      "bytes": 31086,
+      "rows": 430,
+      "columns": 430,
+      "entries": 1544,
+      "download_url": "https://sparse.tamu.edu/MM/Sandia/oscil_dcop_42.tar.gz",
+      "original_archive_sha256": "3a7fce53d2bfbe26dd0bde2995f7b52078ac33359ced79680581ce90fced0b17"
+    },
+    "QCD/conf6_0-4x4-20/conf6_0-4x4-20.mtx": {
+      "sha256": "cffe2924b9f21f869b2ee8fb8d00b042653ac1a62775d5ac40d0b6d1c527a152",
+      "bytes": 5087623,
+      "rows": 3072,
+      "columns": 3072,
+      "entries": 119808,
+      "download_url": "https://sparse.tamu.edu/MM/QCD/conf6_0-4x4-20.tar.gz",
+      "original_archive_sha256": "f6be76ab90502bbd6d645d6cc925ff01efac8d5437927b9e0ebfb57f07b7a4bc"
+    },
+    "QCD/conf6_0-4x4-30/conf6_0-4x4-30.mtx": {
+      "sha256": "5ede088c30cd221bad591af166b76f38e5005e86d9a6f610b3b046815fc95738",
+      "bytes": 5087643,
+      "rows": 3072,
+      "columns": 3072,
+      "entries": 119808,
+      "download_url": "https://sparse.tamu.edu/MM/QCD/conf6_0-4x4-30.tar.gz",
+      "original_archive_sha256": "6c8576ee077339c96747b861a298dc774a1dff7aaf9a1068b91932f89e48ff96"
+    }
+  }
+}
diff --git a/.github/fixtures/matrixmarket.tar.gz b/.github/fixtures/matrixmarket.tar.gz
diff --git a/.github/scripts/.gitignore b/.github/scripts/.gitignore
@@ -0,0 +1 @@
+__pycache__/
diff --git a/.github/scripts/check_matrix_data.py b/.github/scripts/check_matrix_data.py
@@ -0,0 +1,96 @@
+"""Avoid repeated fixture downloads during a SuiteSparse collection outage."""
+
+import hashlib
+import io
+import json
+import pathlib
+import re
+import sys
+import tarfile
+import urllib.request
+
+COLLECTION = "https://sparse.tamu.edu/"
+
+
+def restore_bundle(repo, cache, expected, metadata):
+    archive = repo / ".github/fixtures/matrixmarket.tar.gz"
+    if not archive.exists():
+        return False
+    if metadata is None:
+        raise RuntimeError("Bundled matrix fixtures have no integrity manifest")
+    blob = archive.read_bytes()
+    if hashlib.sha256(blob).hexdigest() != metadata["archive_sha256"]:
+        raise RuntimeError("Bundled matrix archive checksum mismatch")
+    payloads = {}
+    total = 0
+    with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as bundle:
+        members = bundle.getmembers()
+        if len(members) != len(expected) or {m.name for m in members} != expected:
+            raise RuntimeError("Bundled matrix archive contains unexpected paths")
+        for member in members:
+            spec = metadata["files"][member.name]
+            total += member.size
+            if not member.isfile() or member.size != spec["bytes"] or total > 64 * 1024 * 1024:
+                raise RuntimeError("Invalid bundled matrix archive member")
+            payload = bundle.extractfile(member).read()
+            if hashlib.sha256(payload).hexdigest() != spec["sha256"]:
+                raise RuntimeError("Bundled matrix file checksum mismatch")
+            payloads[member.name] = payload
+    for relative, payload in payloads.items():
+        target = cache / relative
+        if not target.resolve().is_relative_to(cache.resolve()):
+            raise RuntimeError("Matrix cache path escapes its root")
+        target.parent.mkdir(parents=True, exist_ok=True)
+        temporary = target.with_suffix(".mtx.tmp")
+        temporary.write_bytes(payload)
+        temporary.replace(target)
+    return True
+
+
+def check_cache(repo, cache, fetch=None):
+    manifest = repo / "test/CMakeModules/download_sparse_datasets.cmake"
+    matrices = re.findall(
+        r'^\s*mtxDownload\("([^"/]+)"\s+"([^"/]+)"\)',
+        manifest.read_text(encoding="utf-8"),
+        re.MULTILINE,
+    )
+    if not matrices:
+        raise RuntimeError("No matrix fixtures found in the CMake manifest")
+    expected = {f"{group}/{name}/{name}.mtx" for name, group in matrices}
+    manifest = repo / ".github/fixtures/matrixmarket.json"
+    metadata = json.loads(manifest.read_text(encoding="utf-8")) if manifest.exists() else None
+    if metadata is not None and set(metadata["files"]) != expected:
+        raise RuntimeError("Bundled matrix manifest does not match the CMake fixture list")
+    missing = []
+    for name, group in matrices:
+        matrix = cache / group / name / (name + ".mtx")
+        if (not matrix.is_file() or matrix.stat().st_size == 0
+                or (metadata is not None and hashlib.sha256(matrix.read_bytes()).hexdigest()
+                    != metadata["files"][f"{group}/{name}/{name}.mtx"]["sha256"])):
+            missing.append(group + "/" + name)
+    if not missing:
+        return "All matrix fixtures are cached; no provider request needed."
+    if restore_bundle(repo, cache, expected, metadata):
+        return "Restored verified bundled matrix fixtures; no provider request needed."
+
+    if fetch is None:
+        def fetch():
+            with urllib.request.urlopen(COLLECTION, timeout=20) as response:
+                return response.read(256 * 1024).decode("utf-8", errors="replace")
+    if "COLLECTION IS OFFLINE" in fetch().upper():
+        raise RuntimeError(
+            "SuiteSparse reports that the collection is offline. "
+            "The matrix fixture cache is incomplete. Restore a complete cache "
+            "or rerun after the provider recovers; matrix tests remain enabled. "
+            + COLLECTION
+        )
+    return "Fixture cache is incomplete; the provider reports no collection outage."
+
+
+if __name__ == "__main__":
+    repo = pathlib.Path(__file__).resolve().parents[2]
+    try:
+        print(check_cache(repo, repo / "build/extern/matrixmarket"))
+    except (OSError, RuntimeError) as error:
+        print(str(error), file=sys.stderr)
+        sys.exit(1)
diff --git a/.github/scripts/test_check_matrix_data.py b/.github/scripts/test_check_matrix_data.py
@@ -0,0 +1,73 @@
+import hashlib
+import io
+import json
+import pathlib
+import tarfile
+import tempfile
+import unittest
+
+from check_matrix_data import check_cache
+
+
+class MatrixCacheTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.repo = pathlib.Path(self.temp.name)
+        manifest = self.repo / "test/CMakeModules/download_sparse_datasets.cmake"
+        manifest.parent.mkdir(parents=True)
+        manifest.write_text('mtxDownload("Trec4" "JGD_Kocay")\n', encoding="utf-8")
+        self.cache = self.repo / "cache"
+
+    def test_complete_cache_works_without_network_during_outage(self):
+        matrix = self.cache / "JGD_Kocay/Trec4/Trec4.mtx"
+        matrix.parent.mkdir(parents=True)
+        matrix.write_text("fixture", encoding="utf-8")
+
+        def no_network():
+            self.fail("A complete cache must not query the provider")
+
+        self.assertIn("cached", check_cache(self.repo, self.cache, no_network))
+
+    def test_empty_fixture_does_not_hide_provider_outage(self):
+        matrix = self.cache / "JGD_Kocay/Trec4/Trec4.mtx"
+        matrix.parent.mkdir(parents=True)
+        matrix.touch()
+        with self.assertRaisesRegex(RuntimeError, "tests remain enabled"):
+            check_cache(self.repo, self.cache, lambda: "The collection is offline")
+
+    def make_bundle(self):
+        directory = self.repo / ".github/fixtures"
+        directory.mkdir(parents=True)
+        archive = directory / "matrixmarket.tar.gz"
+        payload = b"matrix fixture"
+        relative = "JGD_Kocay/Trec4/Trec4.mtx"
+        with tarfile.open(archive, "w:gz") as bundle:
+            member = tarfile.TarInfo(relative)
+            member.size = len(payload)
+            bundle.addfile(member, io.BytesIO(payload))
+        (directory / "matrixmarket.json").write_text(json.dumps({
+            "archive_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),
+            "files": {relative: {"bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest()}}
+        }), encoding="utf-8")
+        return archive, payload
+
+    def test_verified_bundle_restores_without_network(self):
+        _, payload = self.make_bundle()
+
+        def no_network():
+            self.fail("The bundle must work during a provider outage")
+
+        self.assertIn("bundled", check_cache(self.repo, self.cache, no_network))
+        self.assertEqual((self.cache / "JGD_Kocay/Trec4/Trec4.mtx").read_bytes(), payload)
+
+    def test_corrupt_bundle_is_rejected_before_cache_writes(self):
+        archive, _ = self.make_bundle()
+        archive.write_bytes(b"corrupt archive")
+        with self.assertRaisesRegex(RuntimeError, "checksum mismatch"):
+            check_cache(self.repo, self.cache)
+        self.assertFalse(self.cache.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()