Decouple SDK wrapper gate config

Author: YAO-001

examples/optimization/eval_optimize_loop/README.md

@@ -66,17 +66,44 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \
   --output-dir /tmp/eval-optimize-loop-sdk
 ```
 
+Optional wrapper gate and multi-prompt form:
+
+```bash
+python examples/optimization/eval_optimize_loop/run_pipeline.py \
+  --mode sdk \
+  --train path/to/sdk_train.evalset.json \
+  --val path/to/sdk_val.evalset.json \
+  --optimizer-config path/to/sdk_optimizer.json \
+  --gate-config path/to/wrapper_gate.json \
+  --target-prompt system_prompt=prompts/system.md \
+  --target-prompt router_prompt=prompts/router.md \
+  --sdk-call-agent your_package.your_module:call_agent \
+  --output-dir /tmp/eval-optimize-loop-sdk
+```
+
 `--sdk-call-agent` must point to an async callable compatible with
 `AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials
-needed by that callable. SDK mode never silently falls back to fake mode. When
-the SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult`
+needed by that callable. SDK mode never silently falls back to fake mode.
+
+SDK optimizer config and wrapper gate config are intentionally separate.
+`--optimizer-config` is passed unchanged to `AgentOptimizer.optimize`, so it must
+follow the SDK `OptimizeConfigFile` schema. Put wrapper-only gate settings in
+`--gate-config` (for example `{"gate": {"min_val_score_improvement": 0.05,
+"max_total_cost": 1.0}}`). If `--gate-config` is omitted, the wrapper uses the
+same default aggregate gate values as the fake example.
+
+Fake mode is the complete per-case closed loop. SDK mode is the real
+`AgentOptimizer` / `TargetPrompt` path with an aggregate wrapper gate. When the
+SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult`
 aggregate fields into the JSON/Markdown report: baseline/best pass rate,
-pass-rate improvement, metric breakdowns, token usage, duration, LLM cost, and
-round summaries. SDK mode applies an aggregate gate with
+pass-rate improvement, metric breakdowns, token usage, duration, LLM cost,
+all `best_prompts`, and round summaries. SDK mode applies
 `gate_status: partial_applied`: it checks `status == SUCCEEDED`, validation
 improvement against `gate.min_val_score_improvement`, and total LLM cost against
-`gate.max_total_cost`. Per-case checks that require full validation case scores
-are listed in `not_applied_checks` instead of being silently treated as passed.
+`gate.max_total_cost`. Protected-case regression, new-hard-failure, per-case
+delta, and per-case score-drop checks are not claimed in SDK mode unless the SDK
+exposes full per-case validation scores; they are listed in
+`not_applied_checks`.
 
 ## Source Prompt Writes
 

examples/optimization/eval_optimize_loop/eval_loop/backends.py

@@ -54,6 +54,7 @@ class SDKBackend:
     prompt_path: str | Path
     call_agent_path: str | None = None
     update_source: bool = False
+    target_prompt_paths: dict[str, str | Path] | None = None
     last_result: Any | None = None
     last_result_summary: dict[str, Any] | None = None
     last_artifact_dir: str | None = None
@@ -104,7 +105,10 @@ async def optimize_async(
         except Exception as exc:  # pragma: no cover - depends on optional SDK import health
             raise ValueError(f"sdk mode could not import AgentOptimizer/TargetPrompt: {exc}") from exc
 
-        target_prompt = TargetPrompt().add_path("system_prompt", str(self.prompt_path))
+        target_prompt_paths = self._target_prompt_paths()
+        target_prompt = TargetPrompt()
+        for name, path in target_prompt_paths.items():
+            target_prompt.add_path(name, str(path))
         result = await AgentOptimizer.optimize(
             config_path=str(optimizer_config_path),
             call_agent=call_agent,
@@ -121,20 +125,22 @@ async def optimize_async(
         self.last_result = result
         self.last_result_summary = _summarize_sdk_result(result)
         self.last_artifact_dir = str(output_dir)
+        best_prompts = dict(getattr(result, "best_prompts", {}) or {})
+        baseline_prompts = _read_prompt_bundle(target_prompt_paths)
         return [
             CandidatePrompt(
                 candidate_id="sdk_best",
-                prompt=best_prompt,
+                prompt=_render_prompt_bundle(best_prompts),
                 rationale="Best prompt returned by AgentOptimizer.optimize.",
-                prompt_diff=make_unified_diff(
-                    baseline_prompt,
-                    best_prompt,
-                    before_name="baseline_system_prompt.txt",
-                    after_name="sdk_best/system_prompt.txt",
-                ),
+                prompt_diff=_render_prompt_bundle_diff(baseline_prompts, best_prompts),
             )
         ]
 
+    def _target_prompt_paths(self) -> dict[str, str | Path]:
+        if self.target_prompt_paths:
+            return dict(self.target_prompt_paths)
+        return {"system_prompt": self.prompt_path}
+
 
 def _load_call_agent(path: str):
     if ":" not in path:
@@ -171,6 +177,8 @@ def _summarize_sdk_result(result: Any) -> dict[str, Any]:
         "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})),
         "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)),
         "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)),
+        "baseline_prompts": _safe_jsonable(getattr(result, "baseline_prompts", {})),
+        "best_prompts": _safe_jsonable(getattr(result, "best_prompts", {})),
         "rounds": [
             {
                 "validation_pass_rate": _safe_jsonable(getattr(round_record, "validation_pass_rate", None)),
@@ -197,3 +205,40 @@ def _safe_jsonable(value: Any) -> Any:
     if isinstance(value, (str, int, float, bool)) or value is None:
         return value
     return repr(value)
+
+
+def _read_prompt_bundle(paths: dict[str, str | Path]) -> dict[str, str]:
+    return {
+        name: Path(path).read_text(encoding="utf-8")
+        for name, path in paths.items()
+    }
+
+
+def _render_prompt_bundle(prompts: dict[str, str]) -> str:
+    if set(prompts) == {"system_prompt"}:
+        return prompts["system_prompt"]
+    sections = []
+    for name in sorted(prompts):
+        sections.append(f"## {name}\n\n{prompts[name]}")
+    return "\n\n".join(sections)
+
+
+def _render_prompt_bundle_diff(baseline_prompts: dict[str, str], best_prompts: dict[str, str]) -> str:
+    if set(baseline_prompts) == {"system_prompt"} and set(best_prompts) == {"system_prompt"}:
+        return make_unified_diff(
+            baseline_prompts.get("system_prompt", ""),
+            best_prompts.get("system_prompt", ""),
+            before_name="baseline_system_prompt.txt",
+            after_name="sdk_best/system_prompt.txt",
+        )
+    diffs = []
+    for name in sorted(set(baseline_prompts) | set(best_prompts)):
+        diffs.append(
+            make_unified_diff(
+                baseline_prompts.get(name, ""),
+                best_prompts.get(name, ""),
+                before_name=f"baseline/{name}.txt",
+                after_name=f"sdk_best/{name}.txt",
+            )
+        )
+    return "\n\n".join(diffs)

examples/optimization/eval_optimize_loop/eval_loop/report.py

@@ -127,6 +127,20 @@ def render_markdown(report: OptimizationReport) -> str:
         "Update source prompt: "
         + ("yes" if report.run.get("update_source") else "no (default)"),
         "",
+    ])
+    if report.run.get("mode") == "sdk":
+        availability = report.audit.get("sdk_result_availability", {})
+        lines.extend([
+            "SDK mode uses OptimizeResult aggregate validation metrics. "
+            "Full train scores and full per-case validation deltas are not exposed by the SDK result.",
+            "",
+            "SDK availability: "
+            f"aggregate_validation_result={availability.get('aggregate_validation_result')}, "
+            f"full_train_eval_result={availability.get('full_train_eval_result')}, "
+            f"full_per_case_validation_delta={availability.get('full_per_case_validation_delta')}.",
+            "",
+        ])
+    lines.extend([
         "",
         "## Gate Reasons",
         "",