Enforce reference authority in CI with snapshot and conic failure clustering

Author: fraware

.github/workflows/governance-audit.yml

@@ -22,11 +22,18 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r requirements-dev.txt
           python -m pip install -e ".[dev]"
-      - name: Audit registry, manifests, and release metadata
+      - name: Reference authority (index, snapshot, audit, flagship)
+        run: python scripts/reference_authority_check.py
+      - name: Governance evidence tests
         run: |
-          python scripts/refresh_published_run_index.py --check
-          python -m pytest tests/governance/test_host_realistic_publish_evidence.py tests/governance/test_run_host_realistic_publish.py -q
-          python -m conicshield.governance.audit_cli
+          python -m pytest \
+            tests/governance/test_host_realistic_publish_evidence.py \
+            tests/governance/test_reference_authority_invariants.py \
+            tests/governance/test_reference_evidence_tiers.py \
+            tests/governance/test_native_arm_publish_evidence.py \
+            tests/governance/test_run_host_realistic_publish.py \
+            tests/governance/test_conic_suite_report_clusters.py \
+            -q --tb=short
       - name: Export rehearsal matches committed upstream JSON
         run: |
           python scripts/export_inter_sim_offline_graph.py \
@@ -68,5 +75,3 @@ jobs:
             --family-id conicshield-transition-bank-v1 \
             --reason "CI governance rehearsal (governance-audit workflow)" \
             --dry-run
-      - name: Strict governance audit
-        run: python -m conicshield.governance.audit_cli --strict

Makefile

@@ -23,7 +23,7 @@ else
   PYTHON ?= python
 endif
 
-.PHONY: test test-reference test-slow test-solver test-vendor-moreau smoke-solver smoke-check env-check reference-correctness perf-benchmark diff-check trust-dashboard parity-native-licensed artifact-validation-report parity-report audit dashboard validate-fixture lint typecheck format format-check cov cov-gates compile-deps verify-extended bootstrap-moreau upgrade-host-realistic-vendor host-realistic-rehearsal export-upstream-rehearsal batch-solve-report verify-reference-system reference-authority-check refresh-live-upstream-export
+.PHONY: test test-reference test-slow test-solver test-vendor-moreau smoke-solver smoke-check env-check reference-correctness perf-benchmark diff-check trust-dashboard parity-native-licensed artifact-validation-report parity-report audit dashboard validate-fixture lint typecheck format format-check cov cov-gates compile-deps verify-extended bootstrap-moreau upgrade-host-realistic-vendor host-realistic-rehearsal export-upstream-rehearsal batch-solve-report verify-reference-system reference-authority-check reference-authority-snapshot refresh-live-upstream-export
 
 test:
 	$(PYTHON) -m pytest -q
@@ -76,6 +76,9 @@ upgrade-host-realistic-vendor:
 reference-authority-check:
 	$(PYTHON) scripts/reference_authority_check.py
 
+reference-authority-snapshot:
+	$(PYTHON) scripts/generate_reference_authority_snapshot.py
+
 refresh-live-upstream-export:
 	@echo "Usage: $(PYTHON) scripts/refresh_live_upstream_export.py --graph-json <path/to/offline_transition_graph.json>"
 
@@ -85,6 +88,7 @@ verify-reference-system: reference-authority-check
 		tests/governance/test_host_realistic_publish_evidence.py \
 		tests/governance/test_reference_evidence_tiers.py \
 		tests/governance/test_reference_authority_invariants.py \
+		tests/governance/test_conic_suite_report_clusters.py \
 		tests/governance/test_native_arm_publish_evidence.py \
 		tests/governance/test_run_host_realistic_publish.py \
 		tests/governance/test_batch_solve_report.py \

benchmarks/reports/README.md

@@ -4,8 +4,15 @@ Optional JSON reports that illustrate public benchmark evidence shapes. They are
 
 | File | Purpose |
 |------|---------|
+| [`reference_authority_snapshot.json`](reference_authority_snapshot.json) | Committed flagship release alignment (`current_run_id`, gates, provenance); CI `--check` via `reference_authority_check` |
 | [`batch_solve_report.example.json`](batch_solve_report.example.json) | Schema example for `scripts/batch_solve_report.py` output (`native_microbatch` vs `native_compiled_real_batch`) |
 
+Regenerate snapshot after release or flagship bundle changes:
+
+```bash
+make reference-authority-snapshot
+```
+
 ## Regenerate real batch evidence (licensed host)
 
 ```bash

benchmarks/reports/reference_authority_snapshot.json

@@ -0,0 +1,39 @@
+{
+  "schema_version": "conicshield_reference_authority_snapshot/v1",
+  "generated_at_utc": "2026-05-26T04:42:53Z",
+  "family_id": "conicshield-transition-bank-v1",
+  "flagship_run_id": "host-realistic-20260525",
+  "aligned": true,
+  "current_release": {
+    "current_run_id": "host-realistic-20260525",
+    "state": "published",
+    "publishable_arms": [
+      "baseline-unshielded",
+      "shielded-rules-only",
+      "shielded-rules-plus-geometry",
+      "shielded-native-moreau"
+    ],
+    "artifact_gate": "green",
+    "parity_gate": "green",
+    "promotion_gate": "green",
+    "benchmark_bundle_paths": [
+      "benchmarks/published_runs/wsl-native-20260409-091141",
+      "benchmarks/published_runs/wsl-real-20260409-132450",
+      "benchmarks/published_runs/host-realistic-20260525"
+    ]
+  },
+  "flagship_provenance": {
+    "evidence_tier": "vendor_native",
+    "projector_mode": "real_projector",
+    "source_export_json": "benchmarks/external_evidence/offline_graph_export_upstream.json"
+  },
+  "flagship_governance": {
+    "state": "published",
+    "parity_gate": "green",
+    "promotion_gate": "green"
+  },
+  "export_provenance": {
+    "export_kind": "structural_committed",
+    "export_json": "benchmarks/external_evidence/offline_graph_export_upstream.json"
+  }
+}

conicshield/governance/reference_authority.py

@@ -0,0 +1,81 @@
+"""Reference-authority snapshot: flagship release alignment for dashboards and CI."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+_FLAGSHIP_RUN_ID = "host-realistic-20260525"
+_FAMILY_ID = "conicshield-transition-bank-v1"
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def build_reference_authority_snapshot(*, repo_root: Path | None = None) -> dict[str, Any]:
+    """Machine-readable flagship + family release state from committed artifacts."""
+    root = repo_root if repo_root is not None else Path.cwd()
+    current_path = root / "benchmarks" / "releases" / _FAMILY_ID / "CURRENT.json"
+    current = _load_json(current_path)
+    registry = _load_json(root / "benchmarks" / "registry.json")
+    fam = next(f for f in registry["benchmark_families"] if f["family_id"] == _FAMILY_ID)
+
+    flagship_dir = root / "benchmarks" / "published_runs" / _FLAGSHIP_RUN_ID
+    prov = _load_json(flagship_dir / "RUN_PROVENANCE.json") if (flagship_dir / "RUN_PROVENANCE.json").is_file() else {}
+    gov = (
+        _load_json(flagship_dir / "governance_status.json")
+        if (flagship_dir / "governance_status.json").is_file()
+        else {}
+    )
+    export_prov_path = root / "benchmarks" / "external_evidence" / "EXPORT_PROVENANCE.json"
+    export_prov = _load_json(export_prov_path) if export_prov_path.is_file() else {}
+
+    gates_ok = all(current.get(g) == "green" for g in ("artifact_gate", "parity_gate", "promotion_gate"))
+    native_ok = "shielded-native-moreau" in (current.get("publishable_arms") or [])
+    bundle_ok = f"benchmarks/published_runs/{_FLAGSHIP_RUN_ID}" in (current.get("benchmark_bundle_paths") or [])
+    gov_gates_ok = gov and all(gov.get(g) == "green" for g in ("parity_gate", "promotion_gate"))
+    aligned = (
+        current.get("current_run_id") == _FLAGSHIP_RUN_ID
+        and fam.get("current_run_id") == _FLAGSHIP_RUN_ID
+        and current.get("state") == "published"
+        and prov.get("evidence_tier") == "vendor_native"
+        and prov.get("projector_mode") == "real_projector"
+        and gates_ok
+        and native_ok
+        and bundle_ok
+        and gov_gates_ok
+    )
+
+    return {
+        "schema_version": "conicshield_reference_authority_snapshot/v1",
+        "generated_at_utc": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "family_id": _FAMILY_ID,
+        "flagship_run_id": _FLAGSHIP_RUN_ID,
+        "aligned": aligned,
+        "current_release": {
+            "current_run_id": current.get("current_run_id"),
+            "state": current.get("state"),
+            "publishable_arms": current.get("publishable_arms"),
+            "artifact_gate": current.get("artifact_gate"),
+            "parity_gate": current.get("parity_gate"),
+            "promotion_gate": current.get("promotion_gate"),
+            "benchmark_bundle_paths": current.get("benchmark_bundle_paths"),
+        },
+        "flagship_provenance": {
+            "evidence_tier": prov.get("evidence_tier"),
+            "projector_mode": prov.get("projector_mode"),
+            "source_export_json": prov.get("source_export_json"),
+        },
+        "flagship_governance": {
+            "state": gov.get("state"),
+            "parity_gate": gov.get("parity_gate"),
+            "promotion_gate": gov.get("promotion_gate"),
+        },
+        "export_provenance": {
+            "export_kind": export_prov.get("export_kind"),
+            "export_json": export_prov.get("export_json"),
+        },
+    }

conicshield/reference_correctness/report_clusters.py

@@ -0,0 +1,19 @@
+"""Helpers for trusted conic suite reporting (no cvxpy import)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def cluster_cases_by_family(case_rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
+    """Aggregate suite rows by ``family`` for failure dashboards."""
+    clusters: dict[str, dict[str, Any]] = {}
+    for row in case_rows:
+        family = str(row.get("family", "unknown"))
+        bucket = clusters.setdefault(family, {"total": 0, "ok": 0, "failed_case_ids": []})
+        bucket["total"] += 1
+        if row.get("status") == "ok":
+            bucket["ok"] += 1
+        else:
+            bucket["failed_case_ids"].append(row.get("case_id"))
+    return clusters

docs/REFERENCE_AUTHORITY.md

@@ -27,6 +27,7 @@ Orchestration: [`scripts/run_host_realistic_publish.py`](../scripts/run_host_rea
 | Gate | Command |
 |------|---------|
 | Reference authority (index + audit + flagship) | `make reference-authority-check` or `python scripts/reference_authority_check.py` |
+| Refresh committed snapshot | `make reference-authority-snapshot` → `benchmarks/reports/reference_authority_snapshot.json` |
 | Public verification bundle | `make verify-reference-system` |
 | Strict governance audit | `python -m conicshield.governance.audit_cli --strict` |
 

docs/ROADMAP.md

@@ -43,7 +43,7 @@ What is **not** closed or only partially addressed:
 
 1. **Live inter-sim re-export** — `python scripts/refresh_live_upstream_export.py --graph-json <dump.json>` then `make upgrade-host-realistic-vendor` when a patched-host dump is available.
 2. **Shield autograd vs finite differences** — **Decision: defer autograd product claim.** Inter-sim shield **FD** (`tests/vendor/diff/`) and `python scripts/differentiation_check.py --shield-inter-sim` support internal validation only; **autograd / `enable_grad` vs FD** on the production shield QP remains out of scope for external narrative until explicitly promoted.
-3. **Conic suite: failure clustering** — Optional richer **CI artifacts** or dashboards aggregating `conic_suite_report.py` by regime (suite rows already carry case metadata).
+3. **Conic suite: failure clustering** — `conic_suite_report.py` emits `clusters.by_family` and `families_with_failures`; optional committed CI artifacts under `benchmarks/reports/`.
 4. **Physical test tree** — Continue incremental moves per [`tests/STRUCTURE.md`](../tests/STRUCTURE.md); no mass rename required for correctness.
 5. **Second benchmark family** — [`conicshield-shield-qp-micro-v1`](../benchmarks/releases/conicshield-shield-qp-micro-v1/FAMILY_README.md) is scaffold-only (`current_run_id: null`) until you publish a real run and bundles.
 6. **Follow-on product** — Retraining comparisons, richer proof metadata, second benchmark host — each needs a family manifest and governance review.

scripts/conic_suite_report.py

@@ -18,6 +18,7 @@
     raise SystemExit("cvxpy is required: pip install cvxpy") from exc
 
 from conicshield.reference_correctness.conic_suite import run_conic_suite_trusted_only
+from conicshield.reference_correctness.report_clusters import cluster_cases_by_family
 
 
 def _main() -> int:
@@ -37,6 +38,8 @@ def _main() -> int:
     args = p.parse_args()
 
     rows = run_conic_suite_trusted_only(cp, profile=args.profile)
+
+    by_family = cluster_cases_by_family(rows)
     payload: dict[str, Any] = {
         "generated_at_utc": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
         "profile": args.profile,
@@ -47,6 +50,12 @@ def _main() -> int:
             "ok": sum(1 for r in rows if r.get("status") == "ok"),
             "failed": [r.get("case_id") for r in rows if r.get("status") != "ok"],
         },
+        "clusters": {
+            "by_family": by_family,
+            "families_with_failures": sorted(
+                fam for fam, data in by_family.items() if data.get("failed_case_ids")
+            ),
+        },
     }
     text = json.dumps(payload, indent=2)
     if args.out is not None:

scripts/generate_reference_authority_snapshot.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Write benchmarks/reports/reference_authority_snapshot.json from committed release state."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from conicshield.governance.reference_authority import build_reference_authority_snapshot
+
+
+def _repo_root() -> Path:
+    return Path(__file__).resolve().parents[1]
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument(
+        "--out",
+        type=Path,
+        default=None,
+        help="Output path (default: benchmarks/reports/reference_authority_snapshot.json).",
+    )
+    p.add_argument("--check", action="store_true", help="Fail if on-disk file differs from fresh snapshot.")
+    args = p.parse_args()
+
+    root = _repo_root()
+    out = args.out or (root / "benchmarks" / "reports" / "reference_authority_snapshot.json")
+    payload = build_reference_authority_snapshot(repo_root=root)
+    fresh = json.dumps(payload, indent=2) + "\n"
+
+    def _semantic_key(text: str) -> str:
+        data = json.loads(text)
+        data.pop("generated_at_utc", None)
+        return json.dumps(data, indent=2, sort_keys=True)
+
+    if args.check:
+        if not out.is_file():
+            print(f"Missing {out}; run: python {Path(__file__).name}", flush=True)
+            return 2
+        if _semantic_key(out.read_text(encoding="utf-8")) != _semantic_key(fresh):
+            print(f"Stale {out}; run: python {Path(__file__).name}", flush=True)
+            return 2
+        print(f"OK {out}")
+        return 0
+
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(fresh, encoding="utf-8")
+    print(out)
+    if not payload.get("aligned"):
+        print("warning: flagship alignment check failed in snapshot", flush=True)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())