Harden mp-300k gate handoff

Author: MaxGhenis

.github/workflows/mp300k-artifact-gates.yml

@@ -225,7 +225,13 @@ jobs:
                   handle.extractall(extract_root, filter="data")
           elif zipfile.is_zipfile(archive):
               with zipfile.ZipFile(archive) as handle:
-                  handle.extractall(extract_root)
+                  for member in handle.infolist():
+                      destination = (extract_root / member.filename).resolve()
+                      if not destination.is_relative_to(extract_root.resolve()):
+                          raise SystemExit(
+                              f"zip archive member escapes artifact root: {member.filename}"
+                          )
+                      handle.extract(member, extract_root)
           else:
               raise SystemExit("artifact_archive_url must point to a tar or zip archive")
 

src/microplex_us/pipelines/mp300k_artifact_gates.py

@@ -63,7 +63,7 @@ def build_mp300k_artifact_gate_report(
     baseline_dataset = (
         Path(baseline_dataset_path).expanduser()
         if baseline_dataset_path is not None
-        else _manifest_baseline_dataset(manifest)
+        else _manifest_baseline_dataset(artifact_root, manifest)
     )
 
     candidate_gate = _candidate_artifact_gate(
@@ -191,12 +191,17 @@ def _resolve_candidate_dataset_path(
     return dataset_path
 
 
-def _manifest_baseline_dataset(manifest: dict[str, Any]) -> Path | None:
+def _manifest_baseline_dataset(
+    artifact_root: Path, manifest: dict[str, Any]
+) -> Path | None:
     config = dict(manifest.get("config", {}))
     value = config.get("policyengine_baseline_dataset")
     if value is None:
         return None
-    return Path(value).expanduser()
+    baseline_path = Path(value).expanduser()
+    if not baseline_path.is_absolute():
+        baseline_path = artifact_root / baseline_path
+    return baseline_path
 
 
 def _candidate_artifact_gate(
@@ -368,9 +373,27 @@ def _ecps_comparison_gate(
     candidate_loss = summary.get("candidate_enhanced_cps_native_loss")
     baseline_loss = summary.get("baseline_enhanced_cps_native_loss")
     loss_delta = summary.get("enhanced_cps_native_loss_delta")
-    candidate_beats = summary.get("candidate_beats_baseline")
-    if candidate_beats is None and loss_delta is not None:
+    reported_candidate_beats = summary.get("candidate_beats_baseline")
+    details: dict[str, Any] = {}
+    if candidate_loss is not None and baseline_loss is not None:
+        computed_loss_delta = float(candidate_loss) - float(baseline_loss)
+        if (
+            loss_delta is not None
+            and abs(float(loss_delta) - computed_loss_delta) > 1e-12
+        ):
+            details["reported_loss_delta"] = loss_delta
+            details["computed_loss_delta"] = computed_loss_delta
+        loss_delta = computed_loss_delta
+    candidate_beats = None
+    if loss_delta is not None:
         candidate_beats = float(loss_delta) < 0.0
+    if (
+        reported_candidate_beats is not None
+        and candidate_beats is not None
+        and bool(reported_candidate_beats) != candidate_beats
+    ):
+        details["reported_candidate_beats_baseline"] = reported_candidate_beats
+        details["computed_candidate_beats_baseline"] = candidate_beats
     status: GateStatus
     if candidate_beats is None:
         status = "unmeasured"
@@ -393,6 +416,7 @@ def _ecps_comparison_gate(
             "enhanced_cps_native_loss_delta": loss_delta,
             "n_targets_kept": summary.get("n_targets_kept"),
         },
+        details=details,
     )
 
 
@@ -447,24 +471,27 @@ def _runtime_gate(
     if ratio is None and candidate_seconds is not None and baseline_seconds:
         ratio = float(candidate_seconds) / float(baseline_seconds)
     passes = payload.get("passes_runtime_gate")
-    if passes is None and ratio is not None:
-        passes = float(ratio) <= threshold
-    if passes is None:
+    details: dict[str, Any] = {}
+    if ratio is None:
         return _gate(
             "unmeasured",
-            "runtime smoke payload is missing ratio or pass/fail result",
+            "runtime smoke payload is missing ratio or candidate/baseline seconds",
             metrics={
                 "candidate_seconds": candidate_seconds,
                 "baseline_seconds": baseline_seconds,
                 "runtime_ratio": ratio,
                 "runtime_ratio_threshold": threshold,
             },
         )
+    derived_passes = float(ratio) <= threshold
+    if passes is not None and bool(passes) != derived_passes:
+        details["reported_passes_runtime_gate"] = passes
+        details["computed_passes_runtime_gate"] = derived_passes
     return _gate(
-        "pass" if bool(passes) else "fail",
+        "pass" if derived_passes else "fail",
         (
             "candidate runtime is inside the smoke benchmark threshold"
-            if bool(passes)
+            if derived_passes
             else "candidate runtime exceeds the smoke benchmark threshold"
         ),
         metrics={
@@ -473,6 +500,7 @@ def _runtime_gate(
             "runtime_ratio": ratio,
             "runtime_ratio_threshold": threshold,
         },
+        details=details,
     )
 
 

src/microplex_us/pipelines/mp300k_gate_inputs.py

@@ -17,6 +17,7 @@ def package_mp300k_gate_inputs(
     output_dir: str | Path,
     *,
     candidate_dataset_path: str | Path | None = None,
+    baseline_dataset_path: str | Path | None = None,
     ecps_comparison_path: str | Path | None = None,
     runtime_smoke_path: str | Path | None = None,
     benchmark_manifest_path: str | Path | None = None,
@@ -39,6 +40,11 @@ def package_mp300k_gate_inputs(
     )
     if not candidate_dataset.exists():
         raise FileNotFoundError(f"candidate dataset not found: {candidate_dataset}")
+    baseline_dataset = _resolve_baseline_dataset_path(
+        artifact_root,
+        manifest,
+        baseline_dataset_path,
+    )
 
     output_root.mkdir(parents=True, exist_ok=True)
     archive_path = output_root / archive_name
@@ -56,12 +62,26 @@ def package_mp300k_gate_inputs(
     staged_candidate = stage_root / candidate_relpath
     staged_candidate.parent.mkdir(parents=True, exist_ok=True)
     shutil.copy2(candidate_dataset, staged_candidate)
+    baseline_relpath = None
+    if baseline_dataset is not None:
+        if not baseline_dataset.exists():
+            raise FileNotFoundError(f"baseline dataset not found: {baseline_dataset}")
+        baseline_relpath = _baseline_archive_relpath(
+            manifest,
+            baseline_dataset=baseline_dataset,
+            explicit_baseline_path=baseline_dataset_path,
+        )
+        staged_baseline = stage_root / baseline_relpath
+        staged_baseline.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(baseline_dataset, staged_baseline)
 
     staged_manifest = _manifest_for_archive(
         manifest,
         source_artifact_dir=artifact_root,
         source_candidate_dataset=candidate_dataset,
         candidate_relpath=candidate_relpath,
+        source_baseline_dataset=baseline_dataset,
+        baseline_relpath=baseline_relpath,
     )
     _write_json(stage_root / "manifest.json", staged_manifest)
     _write_archive(archive_path, stage_root)
@@ -86,6 +106,9 @@ def package_mp300k_gate_inputs(
         "source_artifact_dir": str(artifact_root.resolve()),
         "source_manifest": _file_descriptor(manifest_path),
         "source_candidate_dataset": _file_descriptor(candidate_dataset),
+        "source_baseline_dataset": (
+            _file_descriptor(baseline_dataset) if baseline_dataset is not None else None
+        ),
         "artifact_archive": _file_descriptor(archive_path),
         "evidence": evidence,
         "workflow_call": {
@@ -124,6 +147,24 @@ def _resolve_candidate_dataset_path(
     return dataset_path
 
 
+def _resolve_baseline_dataset_path(
+    artifact_root: Path,
+    manifest: dict[str, Any],
+    explicit_path: str | Path | None,
+) -> Path | None:
+    if explicit_path is not None:
+        return Path(explicit_path).expanduser()
+    value = dict(manifest.get("config", {})).get("policyengine_baseline_dataset")
+    if value is None:
+        return None
+    if not isinstance(value, str) or not value:
+        raise ValueError("config.policyengine_baseline_dataset must be a path string")
+    baseline_path = Path(value).expanduser()
+    if not baseline_path.is_absolute():
+        baseline_path = artifact_root / baseline_path
+    return baseline_path
+
+
 def _candidate_archive_relpath(
     manifest: dict[str, Any],
     *,
@@ -140,21 +181,48 @@ def _candidate_archive_relpath(
     return Path(candidate_dataset.name)
 
 
+def _baseline_archive_relpath(
+    manifest: dict[str, Any],
+    *,
+    baseline_dataset: Path,
+    explicit_baseline_path: str | Path | None,
+) -> Path:
+    if explicit_baseline_path is not None:
+        return Path("baseline") / baseline_dataset.name
+    value = dict(manifest.get("config", {})).get("policyengine_baseline_dataset")
+    if isinstance(value, str) and value:
+        relpath = Path(value)
+        if not relpath.is_absolute():
+            return relpath
+    return Path("baseline") / baseline_dataset.name
+
+
 def _manifest_for_archive(
     manifest: dict[str, Any],
     *,
     source_artifact_dir: Path,
     source_candidate_dataset: Path,
     candidate_relpath: Path,
+    source_baseline_dataset: Path | None,
+    baseline_relpath: Path | None,
 ) -> dict[str, Any]:
     updated = dict(manifest)
     artifacts = dict(updated.get("artifacts", {}))
     artifacts["policyengine_dataset"] = str(candidate_relpath)
     updated["artifacts"] = artifacts
+    config = dict(updated.get("config", {}))
+    if baseline_relpath is not None:
+        config["policyengine_baseline_dataset"] = str(baseline_relpath)
+    updated["config"] = config
     updated["mp300k_gate_inputs"] = {
         "packaged_at": datetime.now(UTC).isoformat(),
         "source_artifact_dir": str(source_artifact_dir.resolve()),
         "source_candidate_dataset": str(source_candidate_dataset.resolve()),
+        "source_baseline_dataset": (
+            str(source_baseline_dataset.resolve())
+            if source_baseline_dataset is not None
+            else None
+        ),
     }
     return updated
 
@@ -205,6 +273,7 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--artifact-dir", required=True)
     parser.add_argument("--output-dir", required=True)
     parser.add_argument("--candidate-dataset")
+    parser.add_argument("--baseline-dataset")
     parser.add_argument("--ecps-comparison-json")
     parser.add_argument("--runtime-smoke-json")
     parser.add_argument("--benchmark-manifest")
@@ -215,6 +284,7 @@ def main(argv: list[str] | None = None) -> int:
         args.artifact_dir,
         args.output_dir,
         candidate_dataset_path=args.candidate_dataset,
+        baseline_dataset_path=args.baseline_dataset,
         ecps_comparison_path=args.ecps_comparison_json,
         runtime_smoke_path=args.runtime_smoke_json,
         benchmark_manifest_path=args.benchmark_manifest,

tests/pipelines/test_mp300k_artifact_gates.py

@@ -7,6 +7,7 @@
 
 import h5py
 import numpy as np
+import pytest
 
 from microplex_us.pipelines.mp300k_artifact_gates import (
     main,
@@ -258,3 +259,74 @@ def test_ecps_comparison_accepts_existing_broad_loss_array_payload(tmp_path):
         ]
         == 0.25
     )
+
+
+def test_runtime_gate_ignores_contradictory_producer_verdict(tmp_path):
+    artifact_dir = tmp_path / "artifact"
+    artifact_dir.mkdir()
+    _write_minimal_policyengine_dataset(artifact_dir / "candidate.h5")
+    baseline_dataset = _write_minimal_policyengine_dataset(tmp_path / "baseline.h5")
+    benchmark_manifest = tmp_path / "benchmark_manifest.json"
+    benchmark_manifest.write_text(json.dumps({"schema_version": 1}))
+    _write_artifact_manifest(artifact_dir, baseline_dataset=baseline_dataset)
+
+    report_path = write_mp300k_artifact_gate_report(
+        artifact_dir,
+        ecps_comparison_payload={
+            "summary": {
+                "candidate_enhanced_cps_native_loss": 0.1,
+                "baseline_enhanced_cps_native_loss": 0.2,
+            }
+        },
+        runtime_smoke_payload={
+            "runtime_ratio": 10.0,
+            "runtime_ratio_threshold": 1.25,
+            "passes_runtime_gate": True,
+        },
+        benchmark_manifest_path=benchmark_manifest,
+        compute_native_scores=False,
+        update_manifest=False,
+    )
+
+    record = json.loads(report_path.read_text())
+    runtime_gate = record["gates"]["runtime"]
+
+    assert record["summary"]["status"] == "failed"
+    assert runtime_gate["status"] == "fail"
+    assert runtime_gate["details"]["reported_passes_runtime_gate"] is True
+    assert runtime_gate["details"]["computed_passes_runtime_gate"] is False
+
+
+def test_ecps_gate_derives_verdict_from_losses_not_producer_flag(tmp_path):
+    artifact_dir = tmp_path / "artifact"
+    artifact_dir.mkdir()
+    _write_minimal_policyengine_dataset(artifact_dir / "candidate.h5")
+    baseline_dataset = _write_minimal_policyengine_dataset(tmp_path / "baseline.h5")
+    benchmark_manifest = tmp_path / "benchmark_manifest.json"
+    benchmark_manifest.write_text(json.dumps({"schema_version": 1}))
+    _write_artifact_manifest(artifact_dir, baseline_dataset=baseline_dataset)
+
+    report_path = write_mp300k_artifact_gate_report(
+        artifact_dir,
+        ecps_comparison_payload={
+            "summary": {
+                "candidate_enhanced_cps_native_loss": 0.3,
+                "baseline_enhanced_cps_native_loss": 0.2,
+                "enhanced_cps_native_loss_delta": -0.1,
+                "candidate_beats_baseline": True,
+            }
+        },
+        runtime_smoke_payload={"runtime_ratio": 1.0},
+        benchmark_manifest_path=benchmark_manifest,
+        compute_native_scores=False,
+        update_manifest=False,
+    )
+
+    record = json.loads(report_path.read_text())
+    ecps_gate = record["gates"]["ecps_comparison"]
+
+    assert record["summary"]["status"] == "failed"
+    assert ecps_gate["status"] == "fail"
+    assert ecps_gate["metrics"]["enhanced_cps_native_loss_delta"] == pytest.approx(0.1)
+    assert ecps_gate["details"]["reported_candidate_beats_baseline"] is True
+    assert ecps_gate["details"]["computed_candidate_beats_baseline"] is False