fix(eval_optimize_loop): address code review feedback

- run_pipeline.py: REJECT exits 0 by default; added --fail-on-reject for CI
- pipeline.py: live mode validates evalset/optimizer config file existence
- pipeline.py: note cost_usd=0 is intentional (not yet collected from optimizer)
- pipeline.py: add asyncio.wait_for timeout enforcement for live mode
- test_pipeline.py: fix mock to return different baseline/candidate data
- test_models.py: remove dead code (unused path variable, discarded dump)

Author: DNKYr

examples/optimization/eval_optimize_loop/outputs/.gitignore

@@ -1 +1,2 @@
-optimization_report.*
+*
+!.gitignore

examples/optimization/eval_optimize_loop/pipeline.py

@@ -51,11 +51,24 @@ def from_config(
         config = PipelineConfig.model_validate_json(raw)
         cls._resolve_paths(config, config_dir)
 
-        if config.mode == "live" and (call_agent is None or target_prompt is None):
-            raise ValueError(
-                "live mode requires call_agent and target_prompt "
-                "to be passed to from_config()"
-            )
+        if config.mode == "live":
+            if call_agent is None or target_prompt is None:
+                raise ValueError(
+                    "live mode requires call_agent and target_prompt "
+                    "to be passed to from_config()"
+                )
+            missing = []
+            for attr in ("live_train_evalset", "live_val_evalset", "optimizer_config_path"):
+                value = getattr(config, attr, None)
+                if not value:
+                    missing.append(f"{attr} is empty")
+                elif not os.path.isfile(value):
+                    missing.append(f"{attr}: file not found ({value})")
+            if missing:
+                raise ValueError(
+                    "live mode requires valid evalset paths and optimizer config: "
+                    + "; ".join(missing)
+                )
 
         pipeline = cls(config)
         pipeline._live_call_agent = call_agent
@@ -93,10 +106,34 @@ async def run(self) -> PipelineResult:
             fa = attribute_failures(self._extract_case_results(baseline_train))
             candidate_train, candidate_val = await self._evaluate_trace_candidate()
         elif self._config.mode == "live":
-            baseline_train, baseline_val = await self._evaluate_live_baseline()
-            fa = attribute_failures(self._extract_case_results(baseline_train))
-            await self._run_optimization()
-            candidate_train, candidate_val = await self._evaluate_live_candidate()
+            # Enforce pipeline-level timeout via asyncio.wait_for.
+            # AgentOptimizer has its own internal timeout, but this gate-level
+            # limit ensures the entire run() does not exceed budget.
+            timeout = self._config.gate.max_duration_seconds
+            async def _run_live() -> tuple:
+                baseline_train, baseline_val = await self._evaluate_live_baseline()
+                fa = attribute_failures(self._extract_case_results(baseline_train))
+                await self._run_optimization()
+                candidate_train, candidate_val = await self._evaluate_live_candidate()
+                return baseline_train, baseline_val, fa, candidate_train, candidate_val
+
+            try:
+                baseline_train, baseline_val, fa, candidate_train, candidate_val = (
+                    await asyncio.wait_for(_run_live(), timeout=timeout)
+                )
+            except asyncio.TimeoutError:
+                duration = time.monotonic() - t0
+                finished_at = datetime.now(timezone.utc).isoformat()
+                return PipelineResult(
+                    mode=self._config.mode,
+                    gate_decision="REJECT",
+                    gate_reasons=[f"pipeline timed out after {timeout}s"],
+                    duration_seconds=duration,
+                    cost_usd=0.0,
+                    seed=self._config.seed,
+                    started_at=started_at,
+                    finished_at=finished_at,
+                )
         else:
             raise ValueError(f"unknown mode: {self._config.mode}")
 
@@ -112,6 +149,10 @@ async def run(self) -> PipelineResult:
         delta = compute_delta(baseline_split, candidate_split)
 
         duration = time.monotonic() - t0
+        # Note: cost_usd is always 0.0 here because the pipeline does not
+        # yet collect per-round LLM costs from the optimizer.  The gate's
+        # max_cost_usd rule will therefore not reject on cost grounds.
+        # Users should monitor costs via their LLM provider dashboard.
         gate = apply_gate(
             delta, self._config.gate, cost_usd=0.0, duration_seconds=duration
         )

examples/optimization/eval_optimize_loop/run_pipeline.py

@@ -27,6 +27,12 @@ async def main() -> None:
         default=str(_HERE / "pipeline.json"),
         help="Path to pipeline.json (default: pipeline.json in same directory)",
     )
+    parser.add_argument(
+        "--fail-on-reject",
+        action="store_true",
+        help="Exit with code 1 when the pipeline REJECTs the candidate (for CI gating). "
+             "Without this flag, REJECT is considered a valid pipeline outcome and exits 0.",
+    )
     args = parser.parse_args()
 
     pipeline = EvalOptimizePipeline.from_config(args.pipeline_config)
@@ -39,7 +45,7 @@ async def main() -> None:
     print(f"    optimization_report.json")
     print(f"    optimization_report.md")
 
-    if result.gate_decision == "REJECT":
+    if args.fail_on_reject and result.gate_decision == "REJECT":
         sys.exit(1)
 
 

examples/optimization/eval_optimize_loop/tests/test_models.py

@@ -75,8 +75,6 @@ def test_pipeline_result_roundtrip():
         started_at="2026-01-01T00:00:00",
         finished_at="2026-01-01T00:00:05",
     )
-    path = "/tmp/test_pipeline_result.json"
-    result.model_dump_json(indent=2)
     # Verify direct read-back works
     loaded = PipelineResult.model_validate_json(result.model_dump_json())
     assert loaded.mode == "trace"

examples/optimization/eval_optimize_loop/tests/test_pipeline.py

@@ -57,6 +57,10 @@ def test_pipeline_from_config_trace_mode():
 
 def test_pipeline_from_config_live_mode():
     """Pipeline loads live mode config correctly."""
+    Path("/tmp/train.json").touch()
+    Path("/tmp/val.json").touch()
+    Path("/tmp/optimizer.json").touch()
+
     config_data = {
         "mode": "live",
         "live_train_evalset": "/tmp/train.json",
@@ -318,11 +322,21 @@ async def test_run_optimization_calls_injected_hook():
 async def test_run_trace_mode_orchestration():
     pipeline = _make_pipeline()
 
-    fake_train = _make_fake_eval_result("train", ["a", "b"], [True, False])
-    fake_val = _make_fake_eval_result("val", ["c", "d"], [True, True])
+    fake_train_base = _make_fake_eval_result("train_base", ["a", "b"], [False, False])
+    fake_train_cand = _make_fake_eval_result("train_cand", ["a", "b"], [True, False])
+    fake_val_base = _make_fake_eval_result("val_base", ["c", "d"], [True, True])
+    fake_val_cand = _make_fake_eval_result("val_cand", ["c", "d"], [True, False])
 
     async def _fake_run_eval(_path: str):
-        return fake_train if "train" in _path else fake_val
+        if "train_base" in _path:
+            return fake_train_base
+        if "train_cand" in _path:
+            return fake_train_cand
+        if "val_base" in _path:
+            return fake_val_base
+        if "val_cand" in _path:
+            return fake_val_cand
+        raise RuntimeError(f"unexpected path: {_path}")
 
     with patch.object(pipeline, "_run_eval", side_effect=_fake_run_eval):
         with patch("examples.optimization.eval_optimize_loop.pipeline.write_reports"):
@@ -332,8 +346,13 @@ async def _fake_run_eval(_path: str):
     assert result.seed == 42
     assert "train" in result.baseline
     assert "val" in result.baseline
-    assert result.baseline["train"].pass_rate == 0.5
-    assert result.baseline["val"].pass_rate == 1.0
+    assert result.baseline["train"].pass_rate == 0.0   # both fail
+    assert result.baseline["val"].pass_rate == 1.0      # both pass
+    assert result.candidate["train"].pass_rate == 0.5   # one passes now
+    assert result.candidate["val"].pass_rate == 0.5     # one regressed
+    # Delta: train has newly_passing, val has newly_failing
+    assert "a" in result.delta.train.newly_passing
+    assert "d" in result.delta.val.newly_failing
 
 
 @pytest.mark.asyncio