From bd3e814a514d0872dbf0292a7f4184dac8cc0657 Mon Sep 17 00:00:00 2001 From: gHashTag Date: Fri, 31 Jul 2026 17:16:02 +0700 Subject: [PATCH] ci(conformance): gate INDEX_all_formats.json against the packs it summarises The index is what consumers read and what Paper B recomputes its coverage from, and nothing checked that it still agreed with the packs underneath it. That gap already cost a regression. gen_all_formats.py hardcoded the tier for the six wide GoldenFloat rungs -- correct when written, but the rungs later acquired independent second witnesses and were promoted in the pack files. Re-running the generator rewrote the index from 75/0/8 back to 69/6/8, silently reverting an honesty-rule-#10 promotion the packs themselves record. #1576 fixes that instance; this closes the class. Checks: entry names a file that exists; recorded sha256 matches disk; kind agrees with the pack's own bitexact + witnesses state; index witness count matches the pack; header totals match the entries; no pack file missing from the index. Two things are deliberately NOT failures. A bitexact pack with an empty witnesses[] is the normal case -- the ~60 uncontested packs were bit-precise from the start, and witnesses[] records the promotions that were CONTESTED; demanding a witness everywhere would fail the whole corpus and misstate rule #10. And five hand-curated packs carry no bitexact key at all while the index labels them bitexact: an absent flag is unknown, not false, so reading silence as denial would make the gate assert something the pack never said. Both are reported as notes. Follows the wp18_selftest_gate.py convention -- --selftest plants a mutant per check and fails if any survives, so the gate proves itself falsifiable before CI trusts it. All six mutants are killed. Replayed against the live corpus by demoting the six rungs exactly as the old generator would, the gate fails with exit 1 and names each rung with its witness count. Clean on current master, so it stands alone and does not depend on #1576. Closes #1577 --- .../workflows/pack-index-consistency-gate.yml | 58 ++++ tools/pack_index_consistency_gate.py | 288 ++++++++++++++++++ 2 files changed, 346 insertions(+) create mode 100644 .github/workflows/pack-index-consistency-gate.yml create mode 100644 tools/pack_index_consistency_gate.py diff --git a/.github/workflows/pack-index-consistency-gate.yml b/.github/workflows/pack-index-consistency-gate.yml new file mode 100644 index 000000000..e5c6363bb --- /dev/null +++ b/.github/workflows/pack-index-consistency-gate.yml @@ -0,0 +1,58 @@ +# Locks INDEX_all_formats.json against the packs it summarises. +# +# Sibling to conformance-integrity-gate.yml: that gate checks the vectors inside +# each pack; this one checks that the index's summary of a pack still matches the +# pack. The gap it closes is not hypothetical -- gen_all_formats.py hardcoded the +# tier for the six wide GoldenFloat rungs, so re-running it silently rewrote the +# index from 75/0/8 back to 69/6/8, reverting an honesty-rule-#10 promotion the +# pack files themselves record. Nothing in CI would have noticed. +# +# stdlib-only Python, no third-party deps, no network. The self-test runs FIRST and +# must show every check is still FAIL-reachable before the gate is trusted. +name: pack-index-consistency-gate + +on: + push: + paths: + - "conformance/vectors/**" + - "tools/pack_index_consistency_gate.py" + - ".github/workflows/pack-index-consistency-gate.yml" + pull_request: + paths: + - "conformance/vectors/**" + - "tools/pack_index_consistency_gate.py" + - ".github/workflows/pack-index-consistency-gate.yml" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + pack-index-consistency: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Self-test (gate must be falsifiable) + # If any planted mutant survives, the gate is decoration -> fail here. + run: python3 tools/pack_index_consistency_gate.py --selftest + + - name: Run gate against the live corpus + run: | + python3 tools/pack_index_consistency_gate.py \ + --vectors conformance/vectors \ + --json pack_index_report.json + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: pack-index-consistency-report + path: pack_index_report.json + if-no-files-found: ignore diff --git a/tools/pack_index_consistency_gate.py b/tools/pack_index_consistency_gate.py new file mode 100644 index 000000000..3b4790274 --- /dev/null +++ b/tools/pack_index_consistency_gate.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Lock INDEX_all_formats.json against the packs it summarises. + +WHY +--- +Honesty rule #10 -- a pack is not labelled bit-precise without an independent +second witness -- is the corpus's central claim. The pack file is the artefact of +record for its own status; the index is a summary consumers and papers read. +Nothing checked that the two agreed. + +That gap was not hypothetical. gen_all_formats.py hardcoded +`kind: bitexact_selfconsistent` for the six wide GoldenFloat rungs, which was +correct when written. The rungs later acquired independent second witnesses and +were promoted in the pack files; the generator was never updated. Re-running it +rewrote the index from 75/0/8 back to 69/6/8 -- silently reverting an honesty-rule +promotion the packs themselves record, with no gate to catch it. + +This gate closes that class rather than that instance. It is stdlib-only, needs no +network, and is falsifiable: --selftest plants a mutant for every check and fails +if any mutant survives. + +CHECKS +------ + A every index entry names a pack file that exists + B the recorded sha256 matches the file on disk + C kind agrees with the pack: bitexact + witnesses[] => "bitexact", + and a pack that is not bitexact is not labelled anything else. + An empty witnesses[] on a bitexact pack is NOT a failure -- see the note at + the check itself for why demanding one everywhere misstates rule #10. + D the index witness count equals len(pack["witnesses"]) + E the header totals equal the entries actually present + F no pack file in the directory is missing from the index + +Usage: + python3 tools/pack_index_consistency_gate.py [--vectors DIR] [--json OUT] + python3 tools/pack_index_consistency_gate.py --selftest +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DEFAULT_VECTORS = os.path.join(REPO, "conformance", "vectors") +INDEX_NAME = "INDEX_all_formats.json" + + +# --------------------------------------------------------------------------- +# The checks. Each returns a list of failure strings. +# --------------------------------------------------------------------------- +def check(index: dict, packs: dict, on_disk: set, notes: list | None = None) -> list[str]: + """packs maps filename -> parsed pack (or None if unreadable/absent). + + Anything appended to `notes` is reported but does not fail the gate. + """ + if notes is None: + notes = [] + fails = [] + entries = index.get("packs", []) + + for e in entries: + cid, fname = e.get("id", ""), e.get("file", "") + pack = packs.get(fname) + + # A -- the file exists + if pack is None: + fails.append(f"A {cid}: index names {fname!r}, which is not readable") + continue + + # B -- digest matches + recorded = e.get("sha256") + actual = pack["__sha256__"] + if recorded is not None and recorded != actual: + fails.append(f"B {cid}: sha256 {recorded[:12]}... recorded, " + f"{actual[:12]}... on disk") + + # C -- tier agrees with the pack's own state. + # + # An ABSENT bitexact key is unknown, not false. Five hand-curated packs + # predate the flag and carry no such key while the index labels them + # bitexact; reading silence as a denial would turn those into failures and + # the gate would be asserting something the pack never said. They are + # counted as notes instead. + declared = pack.get("bitexact", None) + bitexact = bool(declared) + witnessed = bool(pack.get("witnesses")) + kind = e.get("kind") + if declared is None: + notes.append(f"{cid}: pack declares no bitexact flag; index says " + f"kind={kind!r} (hand-curated pack predating the flag)") + continue + # + # Deliberately NOT checked: a bitexact pack with an empty witnesses[]. + # That is the normal case, not a violation. The 60-odd uncontested packs + # were bit-precise from the start, their independent reference codec being + # part of how the vectors were generated; witnesses[] records the packs + # whose promotion was CONTESTED and had to be argued. A check demanding a + # witness everywhere would fail the whole corpus and would misstate rule + # #10. The asymmetry is the point: a recorded witness must not be ignored, + # but its absence is not evidence of anything. + if bitexact and witnessed: + if kind != "bitexact": + fails.append(f"C {cid}: pack is bitexact with " + f"{len(pack['witnesses'])} witness(es) but index " + f"says kind={kind!r}, not 'bitexact'") + elif not bitexact: + if kind not in ("structural", None): + fails.append(f"C {cid}: pack is not bitexact but index says " + f"kind={kind!r}") + + # D -- witness count agrees + if "witnesses" in e: + want = len(pack.get("witnesses", []) or []) + if e["witnesses"] != want: + fails.append(f"D {cid}: index says {e['witnesses']} witness(es), " + f"pack has {want}") + + # E -- header totals + kinds = [e.get("kind") for e in entries] + for key, want in (("total_packs", len(entries)), + ("bitexact_packs", kinds.count("bitexact")), + ("structural_packs", kinds.count("structural"))): + if key in index and index[key] != want: + fails.append(f"E header {key}={index[key]}, entries give {want}") + if "witnessed_packs" in index: + want = sum(1 for e in entries if e.get("witnesses")) + if index["witnessed_packs"] != want: + fails.append(f"E header witnessed_packs={index['witnessed_packs']}, " + f"entries give {want}") + + # F -- nothing on disk is left out of the index + listed = {e.get("file") for e in entries} + for fname in sorted(on_disk - listed): + fails.append(f"F {fname} is a pack file but appears in no index entry") + + return fails + + +# --------------------------------------------------------------------------- +def load(vectors_dir: str): + index_path = os.path.join(vectors_dir, INDEX_NAME) + if not os.path.exists(index_path): + print(f"no {INDEX_NAME} in {vectors_dir}", file=sys.stderr) + raise SystemExit(3) + index = json.load(open(index_path)) + + packs, on_disk = {}, set() + for fname in os.listdir(vectors_dir): + if not fname.endswith(".json") or fname == INDEX_NAME: + continue + blob = open(os.path.join(vectors_dir, fname), "rb").read() + try: + pack = json.loads(blob) + except json.JSONDecodeError: + continue + # a conformance pack, not some other json living in the directory + if "vectors" not in pack and "bitexact" not in pack: + continue + # instance_*.json carry kind: "instance" -- an instance-level pack for one + # fixed parameterization beneath a catalog-STRUCTURAL family. The index has + # one entry per catalog format, so these belong outside it by design. + if pack.get("kind") == "instance": + continue + pack["__sha256__"] = hashlib.sha256(blob).hexdigest() + packs[fname] = pack + on_disk.add(fname) + return index, packs, on_disk + + +# --------------------------------------------------------------------------- +# Self-test: every check must be reachable, or the gate is decoration. +# --------------------------------------------------------------------------- +def selftest() -> int: + base_index = { + "total_packs": 2, "bitexact_packs": 1, "structural_packs": 1, + "witnessed_packs": 1, + "packs": [ + {"id": "alpha", "file": "alpha.json", "kind": "bitexact", + "sha256": "", "witnesses": 1}, + {"id": "beta", "file": "beta.json", "kind": "structural", + "sha256": "", "witnesses": 0}, + ], + } + base_packs = { + "alpha.json": {"bitexact": True, "witnesses": [{"kind": "sw"}], + "vectors": [], "__sha256__": "aa"}, + "beta.json": {"bitexact": False, "vectors": [], "__sha256__": "bb"}, + } + base_index["packs"][0]["sha256"] = "aa" + base_index["packs"][1]["sha256"] = "bb" + on_disk = {"alpha.json", "beta.json"} + + clean = check(copy.deepcopy(base_index), copy.deepcopy(base_packs), set(on_disk)) + if clean: + print("SELFTEST BROKEN: the clean fixture already fails:") + for f in clean: + print(" ", f) + return 1 + + def mutate(name, letter, fn): + idx, pks, disk = (copy.deepcopy(base_index), copy.deepcopy(base_packs), + set(on_disk)) + fn(idx, pks, disk) + got = check(idx, pks, disk) + killed = any(f.startswith(letter + " ") for f in got) + print(f" [{'PASS' if killed else 'FAIL'}] {name}") + if not killed: + print(f" mutant survived; findings were: {got}") + return 0 if killed else 1 + + def m_missing(i, p, d): + p.pop("alpha.json"); d.discard("alpha.json") + + def m_digest(i, p, d): + p["alpha.json"]["__sha256__"] = "zz" + + def m_demote(i, p, d): + # exactly the pass-48 regression: a witnessed, bitexact pack demoted + i["packs"][0]["kind"] = "bitexact_selfconsistent" + + def m_count(i, p, d): + i["packs"][0]["witnesses"] = 7 + + def m_header(i, p, d): + i["bitexact_packs"] = 99 + + def m_orphan(i, p, d): + p["gamma.json"] = {"bitexact": False, "vectors": [], "__sha256__": "cc"} + d.add("gamma.json") + + print("selftest -- every check must be FAIL-reachable:") + bad = 0 + bad += mutate("A missing pack file is caught", "A", m_missing) + bad += mutate("B digest drift is caught", "B", m_digest) + bad += mutate("C witnessed pack demoted in index is caught", "C", m_demote) + bad += mutate("D witness count mismatch is caught", "D", m_count) + bad += mutate("E header total drift is caught", "E", m_header) + bad += mutate("F pack absent from the index is caught", "F", m_orphan) + + print("\n" + ("selftest OK: the gate is falsifiable." + if not bad else f"selftest FAILED: {bad} mutant(s) survived.")) + return 1 if bad else 0 + + +# --------------------------------------------------------------------------- +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--vectors", default=DEFAULT_VECTORS) + ap.add_argument("--json", dest="json_out") + ap.add_argument("--selftest", action="store_true") + args = ap.parse_args() + + if args.selftest: + return selftest() + + index, packs, on_disk = load(args.vectors) + notes: list[str] = [] + fails = check(index, packs, on_disk, notes) + + print(f"index entries : {len(index.get('packs', []))}") + print(f"pack files : {len(on_disk)}") + print(f"notes : {len(notes)}") + for n in notes: + print(" note:", n) + print(f"failures : {len(fails)}") + for f in fails: + print(" ", f) + + verdict = "CLEAN" if not fails else "FAIL" + print(f"\nverdict: {verdict}") + + if args.json_out: + with open(args.json_out, "w") as fh: + json.dump({"schema": "pack-index-consistency/v1", + "vectors_dir": os.path.abspath(args.vectors), + "entries": len(index.get("packs", [])), + "pack_files": len(on_disk), + "failures": fails, + "verdict": verdict}, fh, indent=2) + return 0 if not fails else 1 + + +if __name__ == "__main__": + raise SystemExit(main())