Add pcs-bench producer gate with release-grade reproducibility ingest.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: fraware

.github/workflows/pcs.yml

@@ -16,10 +16,16 @@ on:
       - "policy/schemas/pcs/RegenerationReport.v0.schema.json"
       - "policy/schemas/pcs/FailureCaseManifest.v0.schema.json"
       - "policy/schemas/pcs/BenchmarkCase.v0.schema.json"
+      - "policy/schemas/pcs/BenchmarkArtifactRef.v0.schema.json"
+      - "policy/schemas/pcs/ReproducibilityBenchmarkManifest.v0.schema.json"
       - "docs/labtrust-benchmark-profile.md"
       - "docs/reference-workflow-template.md"
       - "schemas_or_docs/**"
       - "templates/pcs_workflow_template/**"
+      - "Makefile"
+      - "scripts/pcs_bench_producer.*"
+      - "scripts/generate_pcs_bench_ingest_fixture.py"
+      - "tests/fixtures/pcs_bench_ingest/**"
       - ".github/workflows/pcs.yml"
   pull_request:
     branches: [main]
@@ -36,10 +42,16 @@ on:
       - "policy/schemas/pcs/RegenerationReport.v0.schema.json"
       - "policy/schemas/pcs/FailureCaseManifest.v0.schema.json"
       - "policy/schemas/pcs/BenchmarkCase.v0.schema.json"
+      - "policy/schemas/pcs/BenchmarkArtifactRef.v0.schema.json"
+      - "policy/schemas/pcs/ReproducibilityBenchmarkManifest.v0.schema.json"
       - "docs/labtrust-benchmark-profile.md"
       - "docs/reference-workflow-template.md"
       - "schemas_or_docs/**"
       - "templates/pcs_workflow_template/**"
+      - "Makefile"
+      - "scripts/pcs_bench_producer.*"
+      - "scripts/generate_pcs_bench_ingest_fixture.py"
+      - "tests/fixtures/pcs_bench_ingest/**"
       - ".github/workflows/pcs.yml"
 
 jobs:
@@ -124,6 +136,8 @@ jobs:
         run: python examples/pcs_qc_release/scripts/ci_benchmark_reproducibility.py
       - name: Reproducibility ingest (PcsBenchIngest.v0)
         run: python examples/pcs_qc_release/scripts/ci_validate_benchmark_ingest_golden.py
+      - name: Offline pcs-bench ingest fixture (producer gate)
+        run: python examples/pcs_qc_release/scripts/ci_validate_pcs_bench_ingest_fixture.py
       - name: Benchmark packet fixtures
         run: python examples/pcs_qc_release/scripts/generate_benchmark_packet.py
       - name: Verify LabTrust release protocol package

.gitignore

@@ -133,6 +133,10 @@ audit_self_check/
 # Local reproducibility scratch (not committed)
 _tmp_repro/
 _tmp_repro2/
+_tmp_repro_*/
+_tmp_suite_test/
+benchmark_runs/
+reports/
 
 # OS
 .DS_Store

Makefile

@@ -3,7 +3,7 @@
 # make verify: full verification battery (lint, typecheck, policy, tests, risk-register gate, docs).
 # make paper OUT=<dir>: package-release paper_v0.1 then verify-release (requires OUT= output dir).
 
-.PHONY: test golden bench-smoke lint format typecheck policy-validate no-placeholders e2e-artifacts-chain verification-battery verify paper
+.PHONY: test golden bench-smoke lint format typecheck policy-validate no-placeholders e2e-artifacts-chain verification-battery verify paper pcs-bench-producer
 
 # Default: run fast test suite (no env optional deps for golden/policy)
 test:
@@ -57,3 +57,11 @@ paper:
 	@if [ -z "$${OUT}" ]; then echo "OUT is required (e.g. make paper OUT=./paper_release)"; exit 1; fi; \
 	labtrust package-release --profile paper_v0.1 --out "$$OUT" --seed-base 100 && \
 	labtrust verify-release --release-dir "$$OUT" --strict-fingerprints
+
+# Release-grade reproducibility producer + pcs-bench ingest validation (requires ../pcs-core).
+PCS_CORE ?= ../pcs-core
+PCS_BENCH ?= ../pcs-bench
+BENCH_RUN_DIR ?= benchmark_runs/labtrust_reproducibility
+
+pcs-bench-producer:
+	python scripts/pcs_bench_producer.py

docs/contracts/cli_contract.md

@@ -73,7 +73,7 @@ This document defines the contract for all LabTrust-Gym CLI commands: exit codes
 | regenerate-release-chain | alias for `regenerate-release-protocol` | 0 | same as above | examples/pcs_qc_release/RUNBOOK.md |
 | generate-failure-gallery | `--workflow <id\|property_id>`; `--out <dir>`; optional `--release-dir`; `--workflow-profile` | 0 | Per-case `failure_case_manifest.json`, `README.md`, `artifacts/`, legacy `expected_failure.json` / `repair_hint.json` | examples/pcs_qc_release/failures/ |
 | generate-benchmark-cases | `--workflow <id\|property_id>`; `--out <dir>`; `--seed` (default 42); optional `--release-dir`, `--workflow-profile`; `--pcs-bench-layout`; `--suite-fixture-root`; `--validate-pcs-core-output <pcs-core>`; `--pcs-core-registry <path>` | 0 | Flat: `benchmark_index.json` + per-case dirs. pcs-bench: `suite.yaml`, `benchmark_manifest.v0.json`, `valid/`, `invalid/`, `coverage_report.v0.json` | docs/labtrust-benchmark-profile.md |
-| benchmark-reproducibility | `--workflow` (default hospital_lab.qc_release); `--out <dir>`; `--runs` (default 5); `--seed`; `--mode full_regeneration` (default) or `hash_stability`; optional `--pcs-core`, `--release-dir`, `--certifyedge-bin`, `--validate-pcs-core-output <pcs-core>` | 0 | `benchmark_run.v0.json`, `coverage_report.v0.json`, `benchmark_manifest.v0.json`, `benchmark_report.v0.json`, `regeneration_reports/`, `hash_stability_report.v0.json`, `pcs_bench_ingest.v0.json` | docs/labtrust-benchmark-profile.md |
+| benchmark-reproducibility | `--workflow` (default hospital_lab.qc_release; aliases e.g. `qc-release` resolve to canonical `hospital_lab.qc_release`); `--out <dir>`; `--runs` (default 5); `--seed`; `--mode full_regeneration` (default) or `hash_stability`; optional `--pcs-core`, `--release-dir`, `--certifyedge-bin`, `--validate-pcs-core-output <pcs-core>`; `--release-grade` / `--no-release-grade` (default: release when mode is `full_regeneration`) | 0 | `benchmark_run.v0.json`, `coverage_report.v0.json`, `benchmark_manifest.v0.json` (`evidence_grade`, `certifyedge_live`, `pcs_core_validation`), `benchmark_report.v0.json`, `regeneration_reports/`, `hash_stability_report.v0.json`, `pcs_bench_ingest.v0.json` (embedded runs + `artifact_refs` for all sidecars) | docs/pcs-benchmark-producer.md |
 | verify-benchmark-cases | `--benchmark-dir <dir>`; optional `--pcs-core` (auto-detect); `--validate-pcs-core-output <pcs-core>`; `--json` | 0 | (stderr OK labels) | docs/labtrust-benchmark-profile.md |
 
 **PCS CI scripts (not `labtrust` subcommands):** `run_pcs_ci_local.{sh,ps1}` runs `pytest tests/pcs`, `ci_validate_workflow_profile.py`, `ci_validate_formalization.py`, export/release validation, `check-status-policy`, failure gallery generate/verify, and optional `verify-release-protocol` (matches `.github/workflows/pcs.yml`). `materialize_workflow_profile.py` refreshes `workflow_profile.v0.json` digest. `materialize_formalization_artifacts.py` rebuilds formalization JSON from an existing `release/` tree. `generate_golden.py` regenerates `expected/` in deterministic mode.

docs/pcs-benchmark-producer.md

@@ -37,6 +37,14 @@ python examples/pcs_qc_release/scripts/generate_pcs_bench_suite.py \
 
 ## Reproducibility ingest (pcs-bench runs)
 
+```bash
+make pcs-bench-producer
+# Windows: .\scripts\pcs_bench_producer.ps1
+# Cross-platform: python scripts/pcs_bench_producer.py
+```
+
+Equivalent manual flow:
+
 ```bash
 labtrust benchmark-reproducibility \
   --workflow hospital_lab.qc_release \
@@ -45,15 +53,30 @@ labtrust benchmark-reproducibility \
   --certifyedge-bin certifyedge \
   --runs 5 \
   --out benchmark_runs/labtrust_reproducibility \
-  --validate-pcs-core-output ../pcs-core
+  --validate-pcs-core-output ../pcs-core \
+  --release-grade
 
+pcs-bench validate-ingest \
+  --input benchmark_runs/labtrust_reproducibility/pcs_bench_ingest.v0.json \
+  --pcs-core ../pcs-core
+```
+
+Release-grade mode (`evidence_grade: release` on `benchmark_manifest.v0.json`) requires
+`full_regeneration`, certifyedge success rate 1.0, and per-run pcs-core / release-protocol /
+status-policy validation. `workflow_id` is always canonical (`hospital_lab.qc_release`), even
+when `--workflow` uses a short alias such as `qc-release`.
+
+```bash
 python examples/pcs_qc_release/scripts/publish_reproducibility_ingest.py \
   --out benchmark_runs/labtrust_reproducibility \
   --pcs-core ../pcs-core
 ```
 
 `pcs_bench_ingest.v0.json` embeds pcs-core `BenchmarkRun.v0` (one per iteration) and
-`CoverageReport.v0` (`release_reproducibility_score`). Companion files:
+`CoverageReport.v0` (`release_reproducibility_score`), with pcs-core-compatible
+`artifact_refs` (one ref per embedded object under `artifact_refs/`). LabTrust-extended
+refs (aggregate run, report, manifest, hash stability, regeneration reports) are in
+`benchmark_artifact_refs.labtrust.v0.json`. Companion files:
 
 - `benchmark_run.v0.json` — LabTrust aggregate multi-run summary
 - `benchmark_report.v0.json` — pcs-core suite report with `metric_summaries`
@@ -70,6 +93,7 @@ python examples/pcs_qc_release/scripts/publish_reproducibility_ingest.py \
 | Path | Purpose |
 |------|---------|
 | `examples/pcs_qc_release/benchmark_packet/` | Two-case smoke packet |
+| `tests/fixtures/pcs_bench_ingest/labtrust/pcs_bench_ingest.v0.json` | Offline pcs-bench producer gate (regenerate: `python scripts/generate_pcs_bench_ingest_fixture.py`) |
 | `examples/pcs_qc_release/benchmark_ingest/golden/` | Optional committed ingest golden (regenerate via `materialize_benchmark_ingest_golden.py`) |
 
 See also [labtrust-benchmark-profile.md](labtrust-benchmark-profile.md).

examples/pcs_qc_release/RUNBOOK.md

@@ -331,6 +331,10 @@ python examples/pcs_qc_release/scripts/ci_validate_benchmark_registry_expected.p
 # Minimal two-case packet for pcs-bench smoke (no full gallery):
 python examples/pcs_qc_release/scripts/generate_benchmark_packet.py
 
+# Release-grade producer + pcs-bench ingest validation (from repo root):
+make pcs-bench-producer
+
+# Or manual equivalent:
 labtrust benchmark-reproducibility \
   --workflow hospital_lab.qc_release \
   --pcs-core ../pcs-core \
@@ -339,7 +343,11 @@ labtrust benchmark-reproducibility \
   --mode full_regeneration \
   --out benchmark_runs/labtrust_reproducibility \
   --seed 42 \
-  --validate-pcs-core-output ../pcs-core
+  --validate-pcs-core-output ../pcs-core \
+  --release-grade
+
+# Offline ingest fixture gate (no certifyedge):
+python examples/pcs_qc_release/scripts/ci_validate_pcs_bench_ingest_fixture.py
 
 # Publish ingest for pcs-bench (sibling repo):
 python examples/pcs_qc_release/scripts/publish_reproducibility_ingest.py \

examples/pcs_qc_release/scripts/ci_benchmark_reproducibility.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import json
 import sys
 import tempfile
 from pathlib import Path
@@ -79,12 +80,17 @@ def main() -> int:
         ):
             if not (out / name).is_file():
                 raise FileNotFoundError(f"missing reproducibility output: {name}")
+        manifest = json.loads((out / BENCHMARK_MANIFEST_NAME).read_text(encoding="utf-8"))
+        if manifest.get("evidence_grade") != "developer":
+            raise SystemExit("CI hash_stability run must emit developer evidence_grade")
         if pcs_core_root is not None:
             from labtrust_gym.pcs.bench_schemas import validate_pcs_core_reproducibility_outputs
 
-            validate_pcs_core_reproducibility_outputs(
+            checks = validate_pcs_core_reproducibility_outputs(
                 out, pcs_core_root=pcs_core_root, policy_root=ROOT
             )
+            for label in checks:
+                print(f"  OK {label}")
     print("reproducibility benchmark CI OK")
     return 0
 

examples/pcs_qc_release/scripts/ci_validate_benchmark_ingest_golden.py

@@ -15,6 +15,7 @@
     validate_pcs_core_reproducibility_outputs,
 )
 from labtrust_gym.pcs.benchmark_reproducibility import benchmark_reproducibility
+from labtrust_gym.pcs.workflow_profile import CANONICAL_QC_RELEASE_WORKFLOW_ID
 
 GOLDEN = ROOT / "examples" / "pcs_qc_release" / "benchmark_ingest" / "golden"
 
@@ -43,6 +44,22 @@ def main() -> int:
         checks = validate_pcs_core_reproducibility_outputs(
             out, pcs_core_root=pcs_core, policy_root=ROOT
         )
+        import json
+
+        from labtrust_gym.pcs.benchmark_pcs_bench_ingest import PCS_BENCH_INGEST_NAME
+
+        ingest = json.loads((out / PCS_BENCH_INGEST_NAME).read_text(encoding="utf-8"))
+        if ingest["workflow_id"] != CANONICAL_QC_RELEASE_WORKFLOW_ID:
+            raise SystemExit(
+                f"ingest workflow_id must be {CANONICAL_QC_RELEASE_WORKFLOW_ID!r}"
+            )
+        n_runs = len(ingest.get("benchmark_runs") or [])
+        n_cov = len(ingest.get("coverage_reports") or [])
+        if len(ingest.get("artifact_refs") or []) < n_runs + n_cov:
+            raise SystemExit("ingest must include pcs-core artifact_refs per embedded run/coverage")
+        manifest = json.loads((out / "benchmark_manifest.v0.json").read_text(encoding="utf-8"))
+        if manifest.get("evidence_grade") != "developer":
+            raise SystemExit("hash_stability CI run must use developer evidence_grade")
         for label in checks:
             print(f"  OK {label}")
 

examples/pcs_qc_release/scripts/ci_validate_pcs_bench_ingest_fixture.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""CI: validate committed offline PcsBenchIngest.v0 fixture (pcs-bench producer gate)."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+sys.path.insert(0, str(ROOT / "src"))
+
+from labtrust_gym.pcs.bench_schemas import (
+    resolve_pcs_core_schema_root,
+    validate_producer_ingest_contract,
+)
+
+FIXTURE = ROOT / "tests" / "fixtures" / "pcs_bench_ingest" / "labtrust" / "pcs_bench_ingest.v0.json"
+
+
+def main() -> int:
+    if not FIXTURE.is_file():
+        print(f"FAIL missing fixture: {FIXTURE}", file=sys.stderr)
+        return 1
+
+    doc = json.loads(FIXTURE.read_text(encoding="utf-8"))
+    pcs_core = resolve_pcs_core_schema_root(ROOT.parent / "pcs-core")
+    checks = validate_producer_ingest_contract(
+        doc,
+        ingest_path=FIXTURE,
+        policy_root=ROOT,
+        pcs_core_root=pcs_core,
+    )
+    for label in checks:
+        print(f"  OK {label}")
+    print(f"pcs-bench ingest fixture OK ({FIXTURE.relative_to(ROOT)})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

examples/pcs_qc_release/scripts/run_pcs_ci_local.ps1

@@ -44,6 +44,8 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 & $Python examples/pcs_qc_release/scripts/ci_validate_benchmark_ingest_golden.py
 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+& $Python examples/pcs_qc_release/scripts/ci_validate_pcs_bench_ingest_fixture.py
+if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 
 & $Labtrust check-status-policy --release-dir examples/pcs_qc_release/release
 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }