Add Stage 5 release promotion contract

Author: anth-volk

changelog.d/1037.added

@@ -0,0 +1 @@
+Added the Stage 5 release promotion contract and runtime manifest output.

docs/engineering/stages/release_promotion.md

@@ -121,3 +121,29 @@ cleanup `status` as `skipped`, `completed`, or `failed` on
 `CleanupPromotionResult`. Later contract, index, diagnostics, and status
 writers should read this typed material instead of scraping logs or
 reconstructing public paths independently.
+
+## Release Promotion Contract
+
+Stage 5 writes `release_promotion_contract.json` under the run-local
+`diagnostics/contracts/` directory after the promotion transaction succeeds and
+before the Stage 5 step manifest is completed. The contract is the semantic
+record for the Stage 5 boundary: it ties the canonical `run_id`, candidate
+identity, Stage 4 output contract reference when available, validation report
+paths, public Hugging Face and GCS refs, cleanup status, and typed
+`FullPromotionResult` into one durable `StageContract`.
+
+The contract complements the public release files instead of replacing them:
+
+- `release_manifest.json` and `releases/{version}/release_manifest.json` remain
+  the public artifact inventory for the stable release.
+- `version_manifest.json` remains the public version registry used by clients
+  and publication checks.
+- `releases/{version}/release-complete.json` remains the final completion
+  marker and tag target proving the release was fully finalized.
+- `release_promotion_contract.json` remains run-scoped diagnostics material for
+  dashboards, AI agents, rerun comparison, and promotion auditability.
+
+Runtime step manifests for `5_validate_and_promote_release` should include the
+contract as a JSON `contract` output. They may still record legacy validated
+input artifacts for compatibility, but the contract is the preferred semantic
+entry point for Stage 5 status and lineage.

modal_app/pipeline.py

@@ -524,6 +524,122 @@ def _promote_full_release_from_staging(
     )
 
 
+def _promotion_result_from_stdout(promotion_stdout: str):
+    """Parse typed promotion results from the promotion subprocess output."""
+
+    from policyengine_us_data.release_promotion import FullPromotionResult
+
+    try:
+        payload = json.loads(promotion_stdout)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError(
+            "Full release promotion subprocess did not return JSON."
+        ) from exc
+    return FullPromotionResult.from_legacy_dict(payload)
+
+
+def _release_promotion_context_from_run_context(run_context: RunContext):
+    """Build the Stage 5 library context from the orchestration run context."""
+
+    from policyengine_us_data.release_promotion import ReleasePromotionContext
+
+    return ReleasePromotionContext(
+        run_id=run_context.run_id,
+        candidate_version=run_context.candidate_version,
+        release_version=run_context.release_version,
+        hf_repo_name="policyengine/policyengine-us-data",
+        gcs_bucket_name="policyengine-us-data",
+        base_release_version=run_context.base_release_version or None,
+        release_bump=run_context.release_bump or None,
+        modal_app_name=run_context.modal_app_name or None,
+        modal_environment=run_context.modal_environment or None,
+        hf_staging_prefix=run_context.hf_staging_prefix or None,
+        metadata={"run_context": run_context.to_dict()},
+    )
+
+
+def _release_artifact_metadata_by_path(
+    run_id: str,
+    rel_paths: list[str],
+) -> dict[str, dict[str, object]]:
+    """Return local checksum/size metadata for staged release artifacts."""
+
+    metadata: dict[str, dict[str, object]] = {}
+    for local_path, rel_path in _full_release_manifest_files(run_id, rel_paths):
+        path = Path(local_path)
+        if not path.exists() or not path.is_file():
+            continue
+        reference = ArtifactReference.from_path(path)
+        metadata[rel_path] = {
+            "sha256": f"sha256:{reference.sha256}",
+            "size_bytes": reference.size_bytes,
+        }
+    return metadata
+
+
+def _stage4_output_contract_repo_path_if_available(run_id: str) -> str | None:
+    """Return the run-repo path for the Stage 4 contract when it exists locally."""
+
+    run_dir = _run_dir(run_id)
+    candidates = (
+        run_dir / "diagnostics" / "contracts" / "output_build_contract.json",
+        run_dir / "contracts" / "output_build_contract.json",
+        run_dir / "output_build_contract.json",
+    )
+    for path in candidates:
+        if path.exists() and path.is_file():
+            return f"calibration/runs/{run_id}/{path.relative_to(run_dir).as_posix()}"
+    return None
+
+
+def _write_release_promotion_contract_for_run(
+    *,
+    meta: RunMetadata,
+    run_context: RunContext,
+    rel_paths: list[str],
+    promotion_result,
+) -> ArtifactReference:
+    """Write Stage 5's run-local contract and return its manifest reference."""
+
+    from policyengine_us_data.release_promotion import (
+        build_legacy_release_candidate_bundle,
+        release_promotion_contract_path,
+        write_release_promotion_contract,
+    )
+
+    run_dir = _run_dir(run_context.run_id)
+    contract_path = release_promotion_contract_path(run_dir)
+    candidate_bundle = build_legacy_release_candidate_bundle(
+        context=_release_promotion_context_from_run_context(run_context),
+        rel_paths=rel_paths,
+        artifact_metadata_by_path=_release_artifact_metadata_by_path(
+            run_context.run_id,
+            rel_paths,
+        ),
+        source_output_contract_path=_stage4_output_contract_repo_path_if_available(
+            run_context.run_id
+        ),
+    )
+    write_release_promotion_contract(
+        contract_path=contract_path,
+        candidate_bundle=candidate_bundle,
+        promotion_result=promotion_result,
+        created_at=datetime.now(timezone.utc).isoformat(),
+        code_sha=meta.sha,
+        package_version=meta.version,
+        metadata={
+            "writer": "modal_app.pipeline.promote_run",
+            "branch": meta.branch,
+        },
+    )
+    return ArtifactReference.from_path(
+        contract_path,
+        role="contract",
+        base_dir=run_dir,
+        media_type="application/json",
+    )
+
+
 @app.function(
     image=image,
     timeout=300,
@@ -2013,6 +2129,13 @@ def promote_run(
             promotion_context.to_dict(),
         )
         print(f"  {promotion_stdout}")
+        promotion_result = _promotion_result_from_stdout(promotion_stdout)
+        release_promotion_contract_ref = _write_release_promotion_contract_for_run(
+            meta=meta,
+            run_context=promotion_context,
+            rel_paths=rel_paths,
+            promotion_result=promotion_result,
+        )
 
         # Update run status only after all required promotion work succeeds.
         meta.status = "promoted"
@@ -2021,8 +2144,11 @@ def promote_run(
         _complete_step_manifest(
             promote_manifest,
             outputs=[
-                ArtifactReference.from_dict(artifact)
-                for artifact in promote_inputs["validated_step_outputs"]
+                *[
+                    ArtifactReference.from_dict(artifact)
+                    for artifact in promote_inputs["validated_step_outputs"]
+                ],
+                release_promotion_contract_ref,
             ],
             reuse_decision="computed",
             vol=pipeline_volume,

policyengine_us_data/release_promotion/__init__.py

@@ -24,6 +24,15 @@
     build_release_candidate_bundle_from_stage4_contract,
     read_stage4_release_candidate_bundle,
 )
+from .contract import (
+    RELEASE_PROMOTION_CONTRACT_FILENAME,
+    RELEASE_PROMOTION_CONTRACT_TYPE,
+    ReleasePromotionContractBuilder,
+    build_release_promotion_contract,
+    release_promotion_contract_path,
+    release_promotion_contract_repo_path,
+    write_release_promotion_contract,
+)
 from .results import (
     CleanupPromotionResult,
     CompletionMarkerPromotionResult,
@@ -48,13 +57,16 @@
     "BASE_RELEASE_ARTIFACT_PATHS",
     "DEFAULT_REQUIRED_RELEASE_ARTIFACT_FAMILIES",
     "RELEASE_VALIDATION_SUBSTAGE_ID",
+    "RELEASE_PROMOTION_CONTRACT_FILENAME",
+    "RELEASE_PROMOTION_CONTRACT_TYPE",
     "CleanupPromotionResult",
     "CompletionMarkerPromotionResult",
     "FullPromotionResult",
     "GcsPromotionResult",
     "HuggingFacePromotionResult",
     "ReleaseArtifactSpec",
     "ReleaseCandidateInputBundle",
+    "ReleasePromotionContractBuilder",
     "ReleasePromotionContext",
     "ReleaseCandidateValidationDependencies",
     "ReleaseCandidateValidator",
@@ -63,6 +75,7 @@
     "VALIDATION_REPORT_POLICY_PRESENCE_ONLY",
     "VALIDATION_REPORT_POLICY_REQUIRE_PASSING",
     "build_legacy_release_candidate_bundle",
+    "build_release_promotion_contract",
     "build_release_candidate_bundle_from_stage4_contract",
     "build_release_candidate_shape_report",
     "default_release_candidate_validation_dependencies",
@@ -71,6 +84,9 @@
     "infer_release_artifact_spec",
     "logical_name_for_release_path",
     "normalize_release_path",
+    "release_promotion_contract_path",
+    "release_promotion_contract_repo_path",
     "read_stage4_release_candidate_bundle",
     "strip_staging_prefix",
+    "write_release_promotion_contract",
 ]