fix(eval_optimize_loop): address review feedback for pipeline orchestrator

- Fix Optional[callable] -> Optional[Callable] with typing.Callable import
- Add tests for _write_eval_config_temp, _run_optimization injection hook, and run() trace mode
- Remove unused imports (Path, EvalConfig, EvalSetAggregateResult, GateDecision from pipeline.py)
- Remove unused imports (AgentEvaluator, EvalConfig, PipelineResult, GateConfig from tests)

Author: DNKYr

examples/optimization/eval_optimize_loop/pipeline.py

@@ -4,26 +4,22 @@
 import time
 import tempfile
 from datetime import datetime, timezone
-from pathlib import Path
-from typing import Optional
+from typing import Callable, Optional
 
 from trpc_agent_sdk.evaluation import (
     AgentEvaluator,
     AgentOptimizer,
     CallAgent,
     EvalCaseResult,
-    EvalConfig,
     EvalStatus,
     EvaluateResult,
-    EvalSetAggregateResult,
     TargetPrompt,
 )
 
 from .delta import compute_delta
 from .failure_attribution import attribute_failures
 from .gate import apply_gate
 from .models import (
-    GateDecision,
     PerCaseResult,
     PipelineConfig,
     PipelineResult,
@@ -37,7 +33,7 @@ def __init__(self, config: PipelineConfig) -> None:
         self._config = config
         self._live_call_agent: Optional[CallAgent] = None
         self._live_target_prompt: Optional[TargetPrompt] = None
-        self._optimizer_call: Optional[callable] = None  # type: ignore[valid-type]
+        self._optimizer_call: Optional[Callable] = None
 
     @classmethod
     def from_config(

examples/optimization/eval_optimize_loop/tests/test_pipeline.py

@@ -4,7 +4,7 @@
 import sys
 import tempfile
 from pathlib import Path
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, patch
 
 import pytest
 
@@ -13,9 +13,7 @@
 if str(_EXAMPLE_ROOT.parents[1]) not in sys.path:
     sys.path.insert(0, str(_EXAMPLE_ROOT.parents[1]))
 
-from trpc_agent_sdk.evaluation import AgentEvaluator, EvalConfig
-
-from ..models import PipelineConfig, PipelineResult, GateConfig
+from ..models import PipelineConfig
 from ..pipeline import EvalOptimizePipeline
 
 
@@ -207,3 +205,154 @@ async def test_pipeline_build_split_result():
     assert sr.per_case["case_a"].passed is True
     assert sr.per_case["case_b"].passed is False
     assert "m1" in sr.metric_breakdown
+
+
+# ── Helpers ──────────────────────────────────────────────────────
+
+
+def _make_fake_eval_result(
+    eval_set_id: str,
+    case_ids: list[str],
+    passed: list[bool],
+    metric_name: str = "m1",
+) -> "EvaluateResult":
+    from trpc_agent_sdk.evaluation import (
+        EvalCaseResult,
+        EvalMetricResult,
+        EvalStatus,
+        EvaluateResult,
+        EvalSetAggregateResult,
+    )
+
+    eval_results_by_eval_id: dict[str, list[EvalCaseResult]] = {}
+    for case_id, is_pass in zip(case_ids, passed):
+        status = EvalStatus.PASSED if is_pass else EvalStatus.FAILED
+        score = 1.0 if is_pass else 0.0
+        eval_results_by_eval_id[case_id] = [
+            EvalCaseResult(
+                eval_set_id=eval_set_id,
+                eval_id=case_id,
+                final_eval_status=status,
+                overall_eval_metric_results=[
+                    EvalMetricResult(
+                        metric_name=metric_name,
+                        score=score,
+                        threshold=1.0,
+                        eval_status=status,
+                    ),
+                ],
+                eval_metric_result_per_invocation=[],
+                session_id=f"s_{case_id}",
+            )
+        ]
+
+    return EvaluateResult(
+        results_by_eval_set_id={
+            eval_set_id: EvalSetAggregateResult(
+                eval_results_by_eval_id=eval_results_by_eval_id,
+                num_runs=1,
+            )
+        }
+    )
+
+
+def _make_pipeline(config_overrides: dict | None = None) -> EvalOptimizePipeline:
+    base = {
+        "mode": "trace",
+        "output_dir": "/tmp/fake_outputs",
+        "evaluate": {
+            "metrics": [
+                {
+                    "metric_name": "m1",
+                    "threshold": 1.0,
+                    "criterion": {"final_response": {"text": {"match": "contains"}}},
+                }
+            ],
+            "num_runs": 1,
+        },
+        "train_baseline_evalset": "/tmp/train_base.json",
+        "val_baseline_evalset": "/tmp/val_base.json",
+        "train_candidate_evalset": "/tmp/train_cand.json",
+        "val_candidate_evalset": "/tmp/val_cand.json",
+        "seed": 42,
+    }
+    if config_overrides:
+        base.update(config_overrides)
+
+    pipeline = EvalOptimizePipeline.__new__(EvalOptimizePipeline)
+    pipeline._config = PipelineConfig.model_validate(base)
+    pipeline._live_call_agent = None
+    pipeline._live_target_prompt = None
+    pipeline._optimizer_call = None
+    return pipeline
+
+
+# ── New high-priority tests ─────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_write_eval_config_temp_creates_and_returns_path():
+    pipeline = _make_pipeline()
+    path = await pipeline._write_eval_config_temp()
+    try:
+        assert os.path.isfile(path)
+        import json
+        with open(path) as f:
+            data = json.load(f)
+        assert "metrics" in data
+        assert data["numRuns"] == 1
+    finally:
+        os.unlink(path)
+
+
+@pytest.mark.asyncio
+async def test_run_optimization_calls_injected_hook():
+    pipeline = _make_pipeline({"mode": "live"})
+    hook = AsyncMock()
+    pipeline._optimizer_call = hook
+    await pipeline._run_optimization()
+    hook.assert_called_once_with(pipeline)
+
+
+@pytest.mark.asyncio
+async def test_run_trace_mode_orchestration():
+    pipeline = _make_pipeline()
+
+    fake_train = _make_fake_eval_result("train", ["a", "b"], [True, False])
+    fake_val = _make_fake_eval_result("val", ["c", "d"], [True, True])
+
+    async def _fake_run_eval(_path: str):
+        return fake_train if "train" in _path else fake_val
+
+    with patch.object(pipeline, "_run_eval", side_effect=_fake_run_eval):
+        with patch("examples.optimization.eval_optimize_loop.pipeline.write_reports"):
+            result = await pipeline.run()
+
+    assert result.mode == "trace"
+    assert result.seed == 42
+    assert "train" in result.baseline
+    assert "val" in result.baseline
+    assert result.baseline["train"].pass_rate == 0.5
+    assert result.baseline["val"].pass_rate == 1.0
+
+
+@pytest.mark.asyncio
+async def test_run_optimization_calls_agent_optimizer_when_no_hook():
+    pipeline = _make_pipeline(
+        {
+            "mode": "live",
+            "optimizer_config_path": "/tmp/opt.json",
+            "live_train_evalset": "/tmp/train.json",
+            "live_val_evalset": "/tmp/val.json",
+        }
+    )
+    with patch(
+        "examples.optimization.eval_optimize_loop.pipeline.AgentOptimizer.optimize",
+        new_callable=AsyncMock,
+    ) as mock_optimize:
+        await pipeline._run_optimization()
+
+    mock_optimize.assert_called_once()
+    call_kwargs = mock_optimize.call_args.kwargs
+    assert call_kwargs["config_path"] == "/tmp/opt.json"
+    assert call_kwargs["train_dataset_path"] == "/tmp/train.json"