PCS bench producer contract for qc_release reproducibility.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: fraware

.github/workflows/pcs.yml

@@ -11,6 +11,11 @@ on:
       - "examples/pcs_qc_release/**"
       - "tests/pcs/**"
       - "docs/pcs*.md"
+      - "docs/pcs-bench-producer-contract.md"
+      - "scripts/sync_pcs_bench_labtrust_suite.py"
+      - "scripts/publish_pcs_bench_fixtures.py"
+      - "tests/fixtures/pcs_bench_reproducibility/**"
+      - "examples/pcs_qc_release/scripts/ci_validate_pcs_producer_contract.py"
       - "docs/pcs-workflow-implementation-guide.md"
       - "docs/pcs-reference-implementation.md"
       - "policy/schemas/pcs/RegenerationReport.v0.schema.json"
@@ -37,6 +42,11 @@ on:
       - "examples/pcs_qc_release/**"
       - "tests/pcs/**"
       - "docs/pcs*.md"
+      - "docs/pcs-bench-producer-contract.md"
+      - "scripts/sync_pcs_bench_labtrust_suite.py"
+      - "scripts/publish_pcs_bench_fixtures.py"
+      - "tests/fixtures/pcs_bench_reproducibility/**"
+      - "examples/pcs_qc_release/scripts/ci_validate_pcs_producer_contract.py"
       - "docs/pcs-workflow-implementation-guide.md"
       - "docs/pcs-reference-implementation.md"
       - "policy/schemas/pcs/RegenerationReport.v0.schema.json"
@@ -66,16 +76,21 @@ jobs:
         with:
           repository: SentinelOps-CI/pcs-core
           path: pcs-core
+      - uses: actions/checkout@v4
+        with:
+          repository: SentinelOps-CI/pcs-bench
+          path: pcs-bench
       - name: Apply LabTrust RuntimeReceipt profile to pcs-core schemas
         run: |
           cp policy/schemas/pcs/runtime_receipt.v0.schema.json \
             pcs-core/schemas/RuntimeReceipt.v0.schema.json
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      - name: Install pcs-core and LabTrust-Gym
+      - name: Install pcs-core, pcs-bench, and LabTrust-Gym
         run: |
           pip install -e "pcs-core/python"
+          pip install -e "pcs-bench"
           pip install -e ".[dev,pcs]"
       - name: PCS tests
         run: python -m pytest tests/pcs -q
@@ -138,6 +153,17 @@ jobs:
         run: python examples/pcs_qc_release/scripts/ci_validate_benchmark_ingest_golden.py
       - name: Offline pcs-bench ingest fixture (producer gate)
         run: python examples/pcs_qc_release/scripts/ci_validate_pcs_bench_ingest_fixture.py
+      - name: Materialize offline pcs-bench producer fixtures
+        run: python scripts/generate_pcs_bench_ingest_fixture.py
+      - name: LabTrust producer contract (fixture + pcs-bench validate-ingest)
+        run: |
+          python examples/pcs_qc_release/scripts/ci_validate_pcs_producer_contract.py
+          labtrust validate-pcs-producer \
+            --dir tests/fixtures/pcs_bench_reproducibility \
+            --pcs-core pcs-core
+          pcs-bench validate-ingest \
+            --input tests/fixtures/pcs_bench_reproducibility/pcs_bench_ingest.v0.json \
+            --pcs-core pcs-core
       - name: Benchmark packet fixtures
         run: python examples/pcs_qc_release/scripts/generate_benchmark_packet.py
       - name: Verify LabTrust release protocol package

.gitignore

@@ -136,6 +136,8 @@ _tmp_repro2/
 _tmp_repro_*/
 _tmp_suite_test/
 benchmark_runs/
+!tests/fixtures/pcs_bench_reproducibility/artifact_refs/benchmark_runs/
+!tests/fixtures/pcs_bench_reproducibility/artifact_refs/benchmark_runs/**
 reports/
 
 # OS

Makefile

@@ -3,7 +3,7 @@
 # make verify: full verification battery (lint, typecheck, policy, tests, risk-register gate, docs).
 # make paper OUT=<dir>: package-release paper_v0.1 then verify-release (requires OUT= output dir).
 
-.PHONY: test golden bench-smoke lint format typecheck policy-validate no-placeholders e2e-artifacts-chain verification-battery verify paper pcs-bench-producer
+.PHONY: test golden bench-smoke lint format typecheck policy-validate no-placeholders e2e-artifacts-chain verification-battery verify paper pcs-bench-producer pcs-bench-sync-suite pcs-fixtures pcs-bench-publish-fixtures
 
 # Default: run fast test suite (no env optional deps for golden/policy)
 test:
@@ -65,3 +65,22 @@ BENCH_RUN_DIR ?= benchmark_runs/labtrust_reproducibility
 
 pcs-bench-producer:
 	python scripts/pcs_bench_producer.py
+
+PCS_BENCH_OUT ?= ../pcs-bench/benchmarks/labtrust_qc_release
+
+pcs-bench-sync-suite:
+	python scripts/sync_pcs_bench_labtrust_suite.py --pcs-bench-out $(PCS_BENCH_OUT)
+
+pcs-fixtures:
+	python scripts/generate_pcs_bench_ingest_fixture.py
+
+pcs-bench-publish-fixtures:
+	python scripts/publish_pcs_bench_fixtures.py --pcs-bench $(PCS_BENCH)
+
+pcs-verify: pcs-fixtures
+	python examples/pcs_qc_release/scripts/ci_validate_pcs_bench_ingest_fixture.py
+	python examples/pcs_qc_release/scripts/ci_validate_pcs_producer_contract.py
+	@if [ -d "$(PCS_CORE)" ]; then \
+		labtrust validate-pcs-producer --dir tests/fixtures/pcs_bench_reproducibility --pcs-core $(PCS_CORE); \
+		pcs-bench validate-ingest --input tests/fixtures/pcs_bench_reproducibility/pcs_bench_ingest.v0.json --pcs-core $(PCS_CORE); \
+	fi

docs/contracts/cli_contract.md

@@ -73,7 +73,8 @@ This document defines the contract for all LabTrust-Gym CLI commands: exit codes
 | regenerate-release-chain | alias for `regenerate-release-protocol` | 0 | same as above | examples/pcs_qc_release/RUNBOOK.md |
 | generate-failure-gallery | `--workflow <id\|property_id>`; `--out <dir>`; optional `--release-dir`; `--workflow-profile` | 0 | Per-case `failure_case_manifest.json`, `README.md`, `artifacts/`, legacy `expected_failure.json` / `repair_hint.json` | examples/pcs_qc_release/failures/ |
 | generate-benchmark-cases | `--workflow <id\|property_id>`; `--out <dir>`; `--seed` (default 42); optional `--release-dir`, `--workflow-profile`; `--pcs-bench-layout`; `--suite-fixture-root`; `--validate-pcs-core-output <pcs-core>`; `--pcs-core-registry <path>` | 0 | Flat: `benchmark_index.json` + per-case dirs. pcs-bench: `suite.yaml`, `benchmark_manifest.v0.json`, `valid/`, `invalid/`, `coverage_report.v0.json` | docs/labtrust-benchmark-profile.md |
-| benchmark-reproducibility | `--workflow` (default hospital_lab.qc_release; aliases e.g. `qc-release` resolve to canonical `hospital_lab.qc_release`); `--out <dir>`; `--runs` (default 5); `--seed`; `--mode full_regeneration` (default) or `hash_stability`; optional `--pcs-core`, `--release-dir`, `--certifyedge-bin`, `--validate-pcs-core-output <pcs-core>`; `--release-grade` / `--no-release-grade` (default: release when mode is `full_regeneration`) | 0 | `benchmark_run.v0.json`, `coverage_report.v0.json`, `benchmark_manifest.v0.json` (`evidence_grade`, `certifyedge_live`, `pcs_core_validation`), `benchmark_report.v0.json`, `regeneration_reports/`, `hash_stability_report.v0.json`, `pcs_bench_ingest.v0.json` (embedded runs + `artifact_refs` for all sidecars) | docs/pcs-benchmark-producer.md |
+| benchmark-reproducibility | `--workflow` (default hospital_lab.qc_release; aliases e.g. `qc-release` resolve to canonical `hospital_lab.qc_release`); `--out <dir>`; `--runs` (default 5); `--seed`; `--mode full_regeneration` (default) or `hash_stability`; optional `--pcs-core`, `--release-dir`, `--certifyedge-bin`, `--validate-pcs-core-output <pcs-core>`; `--release-grade` / `--no-release-grade` (default: release when mode is `full_regeneration`) | 0 | `benchmark_run.v0.json`, `coverage_report.v0.json`, `benchmark_manifest.v0.json` (`evidence_grade`, `certifyedge_live`, `pcs_core_validation`, `canonical_hashes_stable`), `benchmark_report.v0.json`, `regeneration_reports/`, `hash_stability_report.v0.json`, `pcs_bench_ingest.v0.json` (embedded runs + `artifact_refs` for all sidecars) | docs/pcs-bench-producer-contract.md |
+| validate-pcs-producer | `--dir <reproducibility-out>`; `--pcs-core` (required) | 0 | Validates ingest contract, pcs-core schemas, on-disk sidecar digests, release-grade manifest when applicable | docs/pcs-bench-producer-contract.md |
 | verify-benchmark-cases | `--benchmark-dir <dir>`; optional `--pcs-core` (auto-detect); `--validate-pcs-core-output <pcs-core>`; `--json` | 0 | (stderr OK labels) | docs/labtrust-benchmark-profile.md |
 
 **PCS CI scripts (not `labtrust` subcommands):** `run_pcs_ci_local.{sh,ps1}` runs `pytest tests/pcs`, `ci_validate_workflow_profile.py`, `ci_validate_formalization.py`, export/release validation, `check-status-policy`, failure gallery generate/verify, and optional `verify-release-protocol` (matches `.github/workflows/pcs.yml`). `materialize_workflow_profile.py` refreshes `workflow_profile.v0.json` digest. `materialize_formalization_artifacts.py` rebuilds formalization JSON from an existing `release/` tree. `generate_golden.py` regenerates `expected/` in deterministic mode.

docs/labtrust-benchmark-profile.md

@@ -151,10 +151,11 @@ unavailable (`examples/pcs_qc_release/scripts/ci_benchmark_reproducibility.py`).
 1. `labtrust generate-benchmark-cases --pcs-bench-layout` emits `suite.yaml`, `benchmark_manifest.v0.json`, `valid/`, `invalid/`, and `coverage_report.v0.json`.
 2. Each case uses `input_artifacts.release_directory` = `input_artifacts/` (flat) or pcs-core fixture paths after layout patch.
 3. Valid cases use pcs-core null failure fields; invalid cases ship `expected_failure.json` and repair hints.
-4. `labtrust benchmark-reproducibility --mode full_regeneration` (default) writes `benchmark_run.v0.json`, `coverage_report.v0.json`, `hash_stability_report.v0.json`, `regeneration_reports/`, `pcs_bench_ingest.v0.json`, and `benchmark_manifest.v0.json`.
-5. `--validate-pcs-core-output ../pcs-core` validates `BenchmarkRun.v0`, `CoverageReport.v0`, and `PcsBenchIngest.v0` (and `BenchmarkReport.v0` when present).
-6. Generated cases validate against pcs-core when a checkout is provided.
-7. pcs-bench ingests `pcs_bench_ingest.v0.json` from reproducibility runs directly.
+4. `make pcs-bench-producer` runs `full_regeneration` (five runs, live CertifyEdge) and `pcs-bench validate-ingest --release-grade`.
+5. `pcs_bench_ingest.v0.json` embeds pcs-core runs/coverage and lists all reproducibility sidecars in `artifact_refs` (see [pcs-bench-producer-contract.md](pcs-bench-producer-contract.md)).
+6. `--validate-pcs-core-output ../pcs-core` validates nested pcs-core schemas; LabTrust checks sidecar digests on disk.
+7. `make pcs-bench-sync-suite` refreshes `../pcs-bench/benchmarks/labtrust_qc_release`.
+8. pcs-bench producer gate consumes `benchmark_runs/labtrust_reproducibility/pcs_bench_ingest.v0.json` without fixture fallback when the producer target succeeds.
 
 ## pcs-bench integration
 

docs/pcs-bench-producer-contract.md

@@ -0,0 +1,179 @@
+# LabTrust PCS bench producer contract
+
+LabTrust-Gym is the reference scientific workflow producer for the hospital lab QC
+release benchmark. It emits a canonical `pcs_bench_ingest.v0.json` aligned with
+[pcs-core](https://github.com/SentinelOps-CI/pcs-core) `PcsBenchIngest.v0` and
+consumable by [pcs-bench](https://github.com/SentinelOps-CI/pcs-bench) without
+fixture fallback.
+
+## Canonical workflow identity
+
+All PCS exports use:
+
+```text
+workflow_id = hospital_lab.qc_release
+```
+
+CLI aliases (`qc_release`, `qc-release`, `labtrust_qc_release`, `hospital_lab_qc_release`,
+`labtrust.qc_release_v0.1`) resolve to this property id before any artifact is written.
+
+## Release-grade producer target
+
+```bash
+make pcs-bench-producer
+```
+
+This runs:
+
+1. `labtrust benchmark-reproducibility` with `full_regeneration`, five runs, live
+   CertifyEdge, pcs-core validation, and release-grade gates.
+2. `pcs-bench validate-ingest --release-grade` on the canonical ingest path.
+
+Environment:
+
+| Variable | Default |
+|----------|---------|
+| `PCS_CORE` | `../pcs-core` |
+| `PCS_BENCH` | `../pcs-bench` |
+| `BENCH_RUN_DIR` | `benchmark_runs/labtrust_reproducibility` |
+
+Manual equivalent:
+
+```bash
+labtrust benchmark-reproducibility \
+  --workflow hospital_lab.qc_release \
+  --mode full_regeneration \
+  --pcs-core ../pcs-core \
+  --certifyedge-bin certifyedge \
+  --runs 5 \
+  --out benchmark_runs/labtrust_reproducibility \
+  --validate-pcs-core-output ../pcs-core \
+  --release-grade
+
+pcs-bench validate-ingest \
+  --input benchmark_runs/labtrust_reproducibility/pcs_bench_ingest.v0.json \
+  --pcs-core ../pcs-core \
+  --release-grade
+```
+
+## Release-grade semantics
+
+A producer output is **release-grade** only when:
+
+| Gate | Requirement |
+|------|-------------|
+| Mode | `full_regeneration` |
+| Runs | `>= 5` |
+| CertifyEdge | `certifyedge_call_success_rate == 1.0` |
+| Per run | `release_protocol_validation_passed`, `status_policy_validation_passed`, `pcs_core_validation_passed` |
+| Stability | `canonical_hashes_stable` and `release_validation_stable` (from `hash_stability_report.v0.json` when `full_regeneration` runs hash-stability copies) |
+| Commit | Real 40-character git `source_commit` (not all zeros) |
+| Embedded objects | Non-empty `benchmark_runs`, non-empty `coverage_reports`, non-empty `commands` |
+| Validation | pcs-core schema + `pcs-bench validate-ingest --release-grade` |
+
+`benchmark_manifest.v0.json` records release evidence:
+
+```json
+{
+  "evidence_grade": "release",
+  "mode": "full_regeneration",
+  "runs": 5,
+  "certifyedge_live": true,
+  "pcs_core_validation": true,
+  "canonical_hashes_stable": true
+}
+```
+
+## Emitted artifacts
+
+Under `benchmark_runs/labtrust_reproducibility/`:
+
+| File | Schema / role |
+|------|----------------|
+| `pcs_bench_ingest.v0.json` | pcs-core `PcsBenchIngest.v0` (embedded runs + coverage + `artifact_refs`) |
+| `benchmark_run.v0.json` | LabTrust aggregate reproducibility summary |
+| `coverage_report.v0.json` | LabTrust reproducibility coverage |
+| `benchmark_report.v0.json` | pcs-core `BenchmarkReport.v0` |
+| `benchmark_manifest.v0.json` | `ReproducibilityBenchmarkManifest.v0` |
+| `hash_stability_report.v0.json` | `HashStabilityReport.v0` |
+| `regeneration_reports/*.json` | `RegenerationReport.v0` per run |
+| `artifact_refs/benchmark_runs/*.v0.json` | pcs-core run sidecars |
+| `artifact_refs/coverage_reports/*.v0.json` | pcs-core coverage sidecar |
+| `runs/run_*` | Regenerated release protocol trees |
+
+### Ingest `artifact_refs`
+
+Embedded pcs-core objects each have a matching sidecar under `artifact_refs/`.
+LabTrust reproducibility sidecars are also listed in ingest `artifact_refs`:
+
+| Path | `artifact_type` | `role` |
+|------|-----------------|--------|
+| `benchmark_run.v0.json` | `LabtrustBenchmarkRunSummary.v0` | `reproducibility_evidence` |
+| `coverage_report.v0.json` | `LabtrustReproducibilityCoverage.v0` | `producer_export` |
+| `benchmark_report.v0.json` | `BenchmarkReport.v0` | `native_report` |
+| `benchmark_manifest.v0.json` | `ReproducibilityBenchmarkManifest.v0` | `producer_export` |
+| `hash_stability_report.v0.json` | `HashStabilityReport.v0` | `reproducibility_evidence` |
+| `regeneration_reports/*.json` | `RegenerationReport.v0` | `regeneration_report` |
+| `pcs_bench_ingest.v0.json` | `PcsBenchIngest.v0` | `canonical_ingest` |
+
+Extended refs are provenance only; pcs-core `BenchmarkRun.v0` and `CoverageReport.v0`
+remain embedded in the ingest body.
+
+## pcs-bench consumption
+
+pcs-bench producer gate resolves:
+
+```text
+benchmark_runs/labtrust_reproducibility/pcs_bench_ingest.v0.json
+```
+
+`pcs-bench validate-ingest` checks:
+
+1. pcs-core `PcsBenchIngest.v0` schema (pcs-core-compatible `artifact_refs` only).
+2. Semantic rules (commit, embedded arrays, digest alignment for embedded types).
+3. Release-grade adequacy (non-empty runs, real commit, sidecar files on disk).
+4. Nested `BenchmarkRun.v0` / `CoverageReport.v0` schemas.
+
+Normalized output feeds the cross-producer gate without using the offline fixture
+when the producer target succeeds.
+
+## Refresh LabTrust pcs-bench fixture suite
+
+```bash
+make pcs-bench-sync-suite
+```
+
+Equivalent manual flow:
+
+```bash
+labtrust generate-benchmark-cases \
+  --workflow hospital_lab.qc_release \
+  --out ../pcs-bench/benchmarks/labtrust_qc_release \
+  --pcs-bench-layout \
+  --seed 42 \
+  --validate-pcs-core-output ../pcs-core
+
+cd ../pcs-bench
+pcs-bench validate-cases --suite labtrust-qc-release --pcs-core ../pcs-core
+```
+
+Produces `suite.yaml`, `valid/`, `invalid/`, `benchmark_manifest.v0.json`, and
+`coverage_report.v0.json` under the pcs-bench benchmarks tree.
+
+## Offline CI fixture
+
+`tests/fixtures/pcs_bench_reproducibility/` is a full developer-grade producer tree
+(ingest, sidecars under `artifact_refs/`, manifest, reports). Regenerate and publish:
+
+```bash
+make pcs-fixtures
+make pcs-verify                   # regenerate + validate fixture + pcs-bench validate-ingest
+make pcs-bench-publish-fixtures   # copies into ../pcs-bench/tests/fixtures and runs/
+```
+
+The committed tree is sidecar-only (no `runs/` scratch); ingest `commands` use repo-relative `--out` paths.
+
+Legacy single-file mirror: `tests/fixtures/pcs_bench_ingest/labtrust/pcs_bench_ingest.v0.json`.
+
+See also [pcs-benchmark-producer.md](pcs-benchmark-producer.md) for command tables
+and CI script references.

docs/pcs-benchmark-producer.md

@@ -4,6 +4,9 @@ LabTrust is the reference producer for the hospital lab QC release PCS benchmark
 machine-readable outputs align with pcs-core v0 schemas and normalize into
 `PcsBenchIngest.v0` for pcs-bench.
 
+See [pcs-bench-producer-contract.md](pcs-bench-producer-contract.md) for the full
+producer contract, release-grade gates, artifact ref roles, and pcs-bench gate paths.
+
 ## Producer surfaces
 
 | Command | Output |

examples/pcs_qc_release/scripts/ci_validate_pcs_bench_ingest_fixture.py

@@ -12,28 +12,46 @@
 
 from labtrust_gym.pcs.bench_schemas import (
     resolve_pcs_core_schema_root,
-    validate_producer_ingest_contract,
+    validate_pcs_core_reproducibility_outputs,
 )
 
-FIXTURE = ROOT / "tests" / "fixtures" / "pcs_bench_ingest" / "labtrust" / "pcs_bench_ingest.v0.json"
+FIXTURE_TREE = ROOT / "tests" / "fixtures" / "pcs_bench_reproducibility"
+LEGACY_INGEST = ROOT / "tests" / "fixtures" / "pcs_bench_ingest" / "labtrust" / "pcs_bench_ingest.v0.json"
 
 
 def main() -> int:
-    if not FIXTURE.is_file():
-        print(f"FAIL missing fixture: {FIXTURE}", file=sys.stderr)
+    tree = FIXTURE_TREE if (FIXTURE_TREE / "pcs_bench_ingest.v0.json").is_file() else None
+    if tree is None and not LEGACY_INGEST.is_file():
+        print(
+            "FAIL missing fixture: run python scripts/generate_pcs_bench_ingest_fixture.py",
+            file=sys.stderr,
+        )
         return 1
 
-    doc = json.loads(FIXTURE.read_text(encoding="utf-8"))
     pcs_core = resolve_pcs_core_schema_root(ROOT.parent / "pcs-core")
-    checks = validate_producer_ingest_contract(
-        doc,
-        ingest_path=FIXTURE,
-        policy_root=ROOT,
-        pcs_core_root=pcs_core,
-    )
-    for label in checks:
-        print(f"  OK {label}")
-    print(f"pcs-bench ingest fixture OK ({FIXTURE.relative_to(ROOT)})")
+    if tree is not None:
+        if pcs_core is None:
+            print("FAIL pcs-core required for fixture tree validation", file=sys.stderr)
+            return 1
+        checks = validate_pcs_core_reproducibility_outputs(
+            tree, pcs_core_root=pcs_core, policy_root=ROOT
+        )
+        for label in checks:
+            print(f"  OK {label}")
+        print(f"pcs-bench reproducibility fixture OK ({tree.relative_to(ROOT)})")
+    elif LEGACY_INGEST.is_file() and pcs_core is not None:
+        from labtrust_gym.pcs.bench_schemas import validate_producer_ingest_contract
+
+        doc = json.loads(LEGACY_INGEST.read_text(encoding="utf-8"))
+        checks = validate_producer_ingest_contract(
+            doc,
+            ingest_path=LEGACY_INGEST,
+            policy_root=ROOT,
+            pcs_core_root=pcs_core,
+        )
+        for label in checks:
+            print(f"  OK {label}")
+        print(f"legacy ingest fixture OK ({LEGACY_INGEST.relative_to(ROOT)})")
     return 0