|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Run v1 reference-system lock checks and print an auditor-facing summary.""" |
| 3 | + |
| 4 | +from __future__ import annotations |
| 5 | + |
| 6 | +import argparse |
| 7 | +import json |
| 8 | +import subprocess |
| 9 | +import sys |
| 10 | +from pathlib import Path |
| 11 | + |
| 12 | + |
| 13 | +def _repo_root() -> Path: |
| 14 | + return Path(__file__).resolve().parents[1] |
| 15 | + |
| 16 | + |
| 17 | +def _run(cmd: list[str], *, cwd: Path, label: str) -> tuple[bool, str]: |
| 18 | + proc = subprocess.run(cmd, cwd=str(cwd), capture_output=True, text=True) |
| 19 | + ok = proc.returncode == 0 |
| 20 | + detail = (proc.stdout or "") + (proc.stderr or "") |
| 21 | + if not detail.strip(): |
| 22 | + detail = f"exit {proc.returncode}" |
| 23 | + return ok, f"{label}: {'OK' if ok else 'FAIL'} — {detail.strip()[:240]}" |
| 24 | + |
| 25 | + |
| 26 | +def main() -> int: |
| 27 | + p = argparse.ArgumentParser(description=__doc__) |
| 28 | + p.add_argument( |
| 29 | + "--full", |
| 30 | + action="store_true", |
| 31 | + help="Also run make verify-reference-system (slow; same as verify-v1-lock).", |
| 32 | + ) |
| 33 | + p.add_argument( |
| 34 | + "--json", |
| 35 | + action="store_true", |
| 36 | + help="Print machine-readable summary on stdout after human report.", |
| 37 | + ) |
| 38 | + args = p.parse_args() |
| 39 | + root = _repo_root() |
| 40 | + py = sys.executable |
| 41 | + results: list[dict[str, object]] = [] |
| 42 | + |
| 43 | + steps: list[tuple[str, list[str]]] = [ |
| 44 | + ("index_integrity", [py, str(root / "scripts" / "refresh_published_run_index.py"), "--check"]), |
| 45 | + ("reference_system_status", [py, str(root / "scripts" / "generate_reference_system_status.py"), "--check"]), |
| 46 | + ("export_cadence", [py, str(root / "scripts" / "check_reference_refresh_cadence.py"), "--max-days", "35"]), |
| 47 | + ("full_refresh_cadence", [py, str(root / "scripts" / "check_flagship_full_refresh_cadence.py"), "--max-days", "35"]), |
| 48 | + ("bundle_profile", [py, str(root / "scripts" / "validate_published_bundle_profile.py")]), |
| 49 | + ( |
| 50 | + "community_verify", |
| 51 | + [ |
| 52 | + py, |
| 53 | + "-m", |
| 54 | + "pytest", |
| 55 | + "tests/test_published_runs_api.py", |
| 56 | + "tests/test_published_runs_cli.py", |
| 57 | + "tests/examples/test_public_examples_smoke.py", |
| 58 | + "-q", |
| 59 | + "--tb=line", |
| 60 | + ], |
| 61 | + ), |
| 62 | + ] |
| 63 | + for name, cmd in steps: |
| 64 | + ok, msg = _run(cmd, cwd=root, label=name) |
| 65 | + results.append({"check": name, "ok": ok}) |
| 66 | + print(msg) |
| 67 | + |
| 68 | + if args.full: |
| 69 | + ok, msg = _run(["make", "verify-reference-system"], cwd=root, label="verify_reference_system") |
| 70 | + results.append({"check": "verify_reference_system", "ok": ok}) |
| 71 | + print(msg) |
| 72 | + else: |
| 73 | + print("skip: verify-reference-system (pass --full or run make verify-v1-lock)") |
| 74 | + |
| 75 | + status_path = root / "benchmarks" / "reports" / "reference_system_status.json" |
| 76 | + if status_path.is_file(): |
| 77 | + status = json.loads(status_path.read_text(encoding="utf-8")) |
| 78 | + print("\nflagship:", status.get("flagship_run_id")) |
| 79 | + print("authority_aligned:", status.get("reference_authority_aligned")) |
| 80 | + print("full_refresh_cadence_ok:", status.get("full_refresh_cadence_ok")) |
| 81 | + community = status.get("community_dataset") or {} |
| 82 | + print("onboarding:", community.get("onboarding_doc")) |
| 83 | + |
| 84 | + all_ok = all(bool(r["ok"]) for r in results) |
| 85 | + if args.json: |
| 86 | + print(json.dumps({"v1_lock_checks": results, "all_ok": all_ok}, indent=2)) |
| 87 | + print("\n" + ("v1 lock checks PASSED" if all_ok else "v1 lock checks FAILED")) |
| 88 | + return 0 if all_ok else 1 |
| 89 | + |
| 90 | + |
| 91 | +if __name__ == "__main__": |
| 92 | + raise SystemExit(main()) |
0 commit comments