fix(eval_optimize_loop): address second code review round

- Add missing import asyncio (live-mode path would NameError)
- Guard _EvaluationCasesFailed private import with ImportError fallback
- Remove /tmp whitelist hack in _resolve_paths (use isabs() only)
- Warn via stderr when max_cost_usd is set but cost tracking unimplemented
- Fix test isolation: use tempfile instead of shared /tmp paths
- Add example tests to CI (testpaths + workflow command)

Author: DNKYr

.github/workflows/ci.yml

@@ -75,7 +75,7 @@ jobs:
 
       - name: Run tests with coverage
         # run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term tests/
-        run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term --cov-fail-under=80 tests/
+        run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term --cov-fail-under=80 tests/ examples/optimization/eval_optimize_loop/tests/
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4

examples/optimization/eval_optimize_loop/pipeline.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+import asyncio
 import os
-import time
+import sys
 import tempfile
+import time
 from datetime import datetime, timezone
 from typing import Callable, Optional
 
@@ -15,7 +17,10 @@
     EvaluateResult,
     TargetPrompt,
 )
-from trpc_agent_sdk.evaluation._agent_evaluator import _EvaluationCasesFailed
+try:
+    from trpc_agent_sdk.evaluation._agent_evaluator import _EvaluationCasesFailed
+except ImportError:
+    _EvaluationCasesFailed = AssertionError
 
 from .delta import compute_delta
 from .failure_attribution import attribute_failures
@@ -73,6 +78,16 @@ def from_config(
         pipeline = cls(config)
         pipeline._live_call_agent = call_agent
         pipeline._live_target_prompt = target_prompt
+
+        if config.gate.max_cost_usd is not None:
+            print(
+                "Note: gate.max_cost_usd is set but cost tracking is not yet "
+                "implemented in the pipeline (cost_usd is always 0.0). "
+                "The cost gate rule will not reject candidates on cost grounds. "
+                "Monitor costs via your LLM provider dashboard.",
+                file=sys.stderr,
+            )
+
         return pipeline
 
     @staticmethod
@@ -89,9 +104,8 @@ def _resolve_paths(config: PipelineConfig, base_dir: str) -> None:
             ("live_val_evalset", config.live_val_evalset),
             ("optimizer_config_path", config.optimizer_config_path),
         ]
-        import tempfile
         for name, value in paths:
-            if value is not None and not os.path.isabs(value) and not value.startswith("/tmp"):
+            if value is not None and not os.path.isabs(value):
                 setattr(config, name, os.path.normpath(os.path.join(base_dir, value)))
 
         if not os.path.isabs(config.output_dir):

examples/optimization/eval_optimize_loop/tests/test_pipeline.py

@@ -57,17 +57,22 @@ def test_pipeline_from_config_trace_mode():
 
 def test_pipeline_from_config_live_mode():
     """Pipeline loads live mode config correctly."""
-    Path("/tmp/train.json").touch()
-    Path("/tmp/val.json").touch()
-    Path("/tmp/optimizer.json").touch()
+    import json
+
+    train_f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+    val_f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+    opt_f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+    train_f.close()
+    val_f.close()
+    opt_f.close()
 
     config_data = {
         "mode": "live",
-        "live_train_evalset": "/tmp/train.json",
-        "live_val_evalset": "/tmp/val.json",
-        "optimizer_config_path": "/tmp/optimizer.json",
+        "live_train_evalset": train_f.name,
+        "live_val_evalset": val_f.name,
+        "optimizer_config_path": opt_f.name,
         "target_prompt_name": "system_prompt",
-        "output_dir": "/tmp/outputs",
+        "output_dir": tempfile.mkdtemp(),
         "evaluate": {
             "metrics": [
                 {
@@ -81,8 +86,6 @@ def test_pipeline_from_config_live_mode():
         "seed": 42,
     }
     with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-        import json
-
         json.dump(config_data, f)
         config_path = f.name
 
@@ -94,6 +97,9 @@ def test_pipeline_from_config_live_mode():
         assert pipeline._live_call_agent is not None
     finally:
         os.unlink(config_path)
+        os.unlink(train_f.name)
+        os.unlink(val_f.name)
+        os.unlink(opt_f.name)
 
 
 def test_pipeline_live_mode_missing_params():

pyproject.toml

@@ -204,5 +204,5 @@ split_before_logical_operator = true
 [tool.pytest.ini_options]
 minversion = "6.0"
 addopts = "-ra -q"
-testpaths = ["tests"]
+testpaths = ["tests", "examples/optimization/eval_optimize_loop/tests"]
 asyncio_mode = "auto"