Map SDK optimize aggregates into eval report

Author: YAO-001

examples/optimization/eval_optimize_loop/README.md

@@ -69,10 +69,14 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \
 `--sdk-call-agent` must point to an async callable compatible with
 `AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials
 needed by that callable. SDK mode never silently falls back to fake mode. When
-the SDK optimizer returns a best prompt but does not expose per-case validation
-results, this example still writes JSON/Markdown reports and audit artifacts
-with `gate_status: not_applicable`, records the SDK result summary, and selects
-`sdk_best` as the SDK optimizer's chosen prompt.
+the SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult`
+aggregate fields into the JSON/Markdown report: baseline/best pass rate,
+pass-rate improvement, metric breakdowns, token usage, duration, LLM cost, and
+round summaries. SDK mode applies an aggregate gate with
+`gate_status: partial_applied`: it checks `status == SUCCEEDED`, validation
+improvement against `gate.min_val_score_improvement`, and total LLM cost against
+`gate.max_total_cost`. Per-case checks that require full validation case scores
+are listed in `not_applied_checks` instead of being silently treated as passed.
 
 ## Source Prompt Writes
 
@@ -110,7 +114,8 @@ optional `simulated_outputs`; it does not depend on sample `case_id` names.
 - failure attribution summary and attribution accuracy when expected labels are
   present;
 - gate decisions with overfit detection, protected regressions, new hard
-  failures, excessive drops, and cost fields;
+  failures, excessive drops, cost fields, and SDK `not_applied_checks` when
+  per-case validation details are not exposed;
 - audit data: seed, duration, config hash, input hashes, candidate prompt hashes,
   cost, prompt diffs, and reproducibility command.
 

examples/optimization/eval_optimize_loop/eval_loop/backends.py

@@ -54,6 +54,7 @@ class SDKBackend:
     prompt_path: str | Path
     call_agent_path: str | None = None
     update_source: bool = False
+    last_result: Any | None = None
     last_result_summary: dict[str, Any] | None = None
     last_artifact_dir: str | None = None
 
@@ -117,6 +118,7 @@ async def optimize_async(
         best_prompt = getattr(result, "best_prompts", {}).get("system_prompt")
         if not best_prompt:
             raise ValueError("sdk mode completed but OptimizeResult.best_prompts['system_prompt'] was missing")
+        self.last_result = result
         self.last_result_summary = _summarize_sdk_result(result)
         self.last_artifact_dir = str(output_dir)
         return [
@@ -157,16 +159,37 @@ def _has_running_loop() -> bool:
 
 
 def _summarize_sdk_result(result: Any) -> dict[str, Any]:
-    if hasattr(result, "model_dump"):
-        payload = result.model_dump(mode="json")
-    elif hasattr(result, "__dict__"):
-        payload = dict(result.__dict__)
-    else:
-        payload = {"repr": repr(result)}
-    return _safe_jsonable(payload)
+    return {
+        "status": _safe_jsonable(getattr(result, "status", None)),
+        "baseline_pass_rate": _safe_jsonable(getattr(result, "baseline_pass_rate", None)),
+        "best_pass_rate": _safe_jsonable(getattr(result, "best_pass_rate", None)),
+        "pass_rate_improvement": _safe_jsonable(getattr(result, "pass_rate_improvement", None)),
+        "baseline_metric_breakdown": _safe_jsonable(getattr(result, "baseline_metric_breakdown", {})),
+        "best_metric_breakdown": _safe_jsonable(getattr(result, "best_metric_breakdown", {})),
+        "metric_thresholds": _safe_jsonable(getattr(result, "metric_thresholds", {})),
+        "total_llm_cost": _safe_jsonable(getattr(result, "total_llm_cost", 0.0)),
+        "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})),
+        "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)),
+        "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)),
+        "rounds": [
+            {
+                "validation_pass_rate": _safe_jsonable(getattr(round_record, "validation_pass_rate", None)),
+                "accepted": _safe_jsonable(getattr(round_record, "accepted", None)),
+                "failed_case_ids": _safe_jsonable(getattr(round_record, "failed_case_ids", [])),
+                "round_llm_cost": _safe_jsonable(getattr(round_record, "round_llm_cost", 0.0)),
+                "budget_used": _safe_jsonable(getattr(round_record, "budget_used", None)),
+                "budget_total": _safe_jsonable(getattr(round_record, "budget_total", None)),
+            }
+            for round_record in getattr(result, "rounds", []) or []
+        ],
+    }
 
 
 def _safe_jsonable(value: Any) -> Any:
+    if hasattr(value, "model_dump"):
+        return _safe_jsonable(value.model_dump(mode="json"))
+    if hasattr(value, "__dict__") and not isinstance(value, type):
+        return _safe_jsonable(dict(value.__dict__))
     if isinstance(value, dict):
         return {str(key): _safe_jsonable(item) for key, item in value.items()}
     if isinstance(value, (list, tuple)):

examples/optimization/eval_optimize_loop/eval_loop/report.py

@@ -140,6 +140,8 @@ def render_markdown(report: OptimizationReport) -> str:
         lines.append(f"### {decision.candidate_id} ({verdict})")
         for reason in decision.reasons:
             lines.append(f"- {reason}")
+        if decision.not_applied_checks:
+            lines.append(f"- not applied checks: {', '.join(decision.not_applied_checks)}")
         lines.append("")
 
     lines.extend([

examples/optimization/eval_optimize_loop/eval_loop/schemas.py

@@ -122,6 +122,7 @@ class GateDecision:
     cost: float
     gate_status: str = "applied"
     gate_not_applied_reason: str | None = None
+    not_applied_checks: list[str] = field(default_factory=list)
 
 
 @dataclass(frozen=True)

examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json

@@ -1060,6 +1060,7 @@
         "val_explain_cache",
         "val_protected_yes_no"
       ],
+      "not_applied_checks": [],
       "overfit_detected": true,
       "protected_regressions": [
         "val_protected_yes_no"
@@ -1089,6 +1090,7 @@
       "gate_not_applied_reason": null,
       "gate_status": "applied",
       "new_hard_failures": [],
+      "not_applied_checks": [],
       "overfit_detected": false,
       "protected_regressions": [],
       "reasons": [

examples/optimization/eval_optimize_loop/run_pipeline.py

@@ -67,6 +67,7 @@ def run_pipeline(
     if mode == "sdk":
         optimizer_config_dict = _read_json_object_for_audit(optimizer_config_path)
         baseline_prompt = load_prompt(prompt_path)
+        sdk_artifact_dir = Path(output_dir) / "sdk_optimizer"
         sdk_backend = SDKBackend(
             prompt_path=prompt_path,
             call_agent_path=sdk_call_agent,
@@ -77,7 +78,7 @@ def run_pipeline(
             train_path=train_path,
             val_path=val_path,
             optimizer_config_path=optimizer_config_path,
-            output_dir=output_dir,
+            output_dir=sdk_artifact_dir,
         )
         report = _build_sdk_report(
             candidates=candidates,
@@ -329,12 +330,24 @@ def _build_sdk_report(
         optimizer_config_path=optimizer_config_path,
         prompt_path=prompt_path,
     )
+    sdk_summary = sdk_backend.last_result_summary or {}
+    baseline_pass_rate = _summary_float(sdk_summary, "baseline_pass_rate", 0.0)
+    best_pass_rate = _summary_float(sdk_summary, "best_pass_rate", baseline_pass_rate)
+    pass_rate_improvement = _summary_float(
+        sdk_summary,
+        "pass_rate_improvement",
+        best_pass_rate - baseline_pass_rate,
+    )
+    total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0)
+    duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0)
+    gate_config = _sdk_gate_config(optimizer_config_dict)
+
     baseline_train = EvalResult(prompt_id="baseline", split="train", score=0.0, passed=False, cost=0.0, cases=[])
     baseline_validation = EvalResult(
         prompt_id="baseline",
         split="validation",
-        score=0.0,
-        passed=False,
+        score=baseline_pass_rate,
+        passed=baseline_pass_rate >= 1.0,
         cost=0.0,
         cases=[],
     )
@@ -352,14 +365,14 @@ def _build_sdk_report(
             "validation_result": EvalResult(
                 prompt_id=candidate.candidate_id,
                 split="validation",
-                score=0.0,
-                passed=False,
-                cost=0.0,
+                score=best_pass_rate,
+                passed=best_pass_rate >= baseline_pass_rate,
+                cost=total_llm_cost,
                 cases=[],
             ),
-            "gate_status": "not_applicable",
-            "gate_not_applied_reason": "SDK optimizer result does not expose per-case validation results",
-            "sdk_result_summary": sdk_backend.last_result_summary or {},
+            "gate_status": "partial_applied",
+            "gate_not_applied_reason": "SDK OptimizeResult exposes aggregate scores but not full per-case deltas",
+            "sdk_result_summary": sdk_summary,
         }
         for candidate in candidates
     ]
@@ -369,7 +382,7 @@ def _build_sdk_report(
     }
     audit = {
         "seed": None,
-        "duration_seconds": 0.0,
+        "duration_seconds": duration_seconds,
         "config_hash": stable_config_hash(optimizer_config_dict),
         "input_hashes": input_hashes,
         "input_paths": {
@@ -380,8 +393,12 @@ def _build_sdk_report(
         },
         "prompt_hash": input_hashes["prompt"],
         "candidate_prompt_hashes": prompt_hashes,
-        "total_run_cost": 0.0,
-        "cost": {"baseline": 0.0, "candidates": {candidate.candidate_id: 0.0 for candidate in candidates}, "total": 0.0},
+        "total_run_cost": total_llm_cost,
+        "cost": {
+            "baseline": 0.0,
+            "candidates": {candidate.candidate_id: total_llm_cost for candidate in candidates},
+            "total": total_llm_cost,
+        },
         "candidate_prompts": {
             candidate.candidate_id: {
                 "rationale": candidate.rationale,
@@ -391,8 +408,8 @@ def _build_sdk_report(
             for candidate in candidates
         },
         "prompt_diffs": {candidate.candidate_id: candidate.prompt_diff for candidate in candidates},
-        "sdk_artifact_dir": str(output_dir),
-        "sdk_result_summary": sdk_backend.last_result_summary or {},
+        "sdk_artifact_dir": sdk_backend.last_artifact_dir or str(Path(output_dir) / "sdk_optimizer"),
+        "sdk_result_summary": sdk_summary,
         "reproducibility_command": _sdk_reproducibility_command(
             train_path=train_path,
             val_path=val_path,
@@ -411,7 +428,7 @@ def _build_sdk_report(
         "train_cases": train_case_count,
         "validation_cases": validation_case_count,
         "update_source": update_source,
-        "sdk_artifact_dir": str(output_dir),
+        "sdk_artifact_dir": audit["sdk_artifact_dir"],
         "reproducibility_command": audit["reproducibility_command"],
         "paths": {
             "train": str(train_path),
@@ -422,34 +439,24 @@ def _build_sdk_report(
         "prompt_source": str(prompt_path),
     }
     gate_decisions = [
-        GateDecision(
+        _sdk_gate_decision(
             candidate_id=candidate.candidate_id,
-            accepted=True,
-            reasons=["gate not applied: SDK optimizer result does not expose per-case validation results"],
-            train_score_delta=0.0,
-            validation_score_delta=0.0,
-            new_hard_failures=[],
-            protected_regressions=[],
-            validation_new_failures=[],
-            excessive_score_drops=[],
-            overfit_detected=False,
-            candidate_cost=0.0,
-            cumulative_cost=0.0,
-            total_run_cost=0.0,
-            cost=0.0,
-            gate_status="not_applicable",
-            gate_not_applied_reason="SDK optimizer result does not expose per-case validation results",
+            sdk_summary=sdk_summary,
+            gate_config=gate_config,
         )
         for candidate in candidates
     ]
+    selected_candidate = None
+    if candidates and gate_decisions and gate_decisions[0].accepted:
+        selected_candidate = candidates[0].candidate_id
     return build_report(
         run=run,
         baseline_train=baseline_train,
         baseline_validation=baseline_validation,
         candidate_records=candidate_records,
         per_case_deltas=[],
         gate_decisions=gate_decisions,
-        selected_candidate=candidates[0].candidate_id if candidates else None,
+        selected_candidate=selected_candidate,
         audit=audit,
     )
 
@@ -474,6 +481,100 @@ def _sdk_reproducibility_command(
     return command
 
 
+def _sdk_gate_decision(
+    *,
+    candidate_id: str,
+    sdk_summary: dict[str, Any],
+    gate_config: dict[str, float],
+) -> GateDecision:
+    status = str(sdk_summary.get("status") or "UNKNOWN")
+    improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0)
+    total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0)
+    min_improvement = gate_config["min_val_score_improvement"]
+    max_cost = gate_config["max_total_cost"]
+
+    reasons: list[str] = []
+    accepted = True
+    if status != "SUCCEEDED":
+        accepted = False
+        reasons.append(f"reject: SDK optimizer status {status} is not SUCCEEDED")
+    else:
+        reasons.append("accept: SDK optimizer status is SUCCEEDED")
+
+    if improvement < min_improvement:
+        accepted = False
+        reasons.append(
+            f"reject: validation improvement {improvement:.3f} is below threshold {min_improvement:.3f}"
+        )
+    else:
+        reasons.append(
+            f"accept: validation improvement {improvement:.3f} meets threshold {min_improvement:.3f}"
+        )
+
+    if total_cost > max_cost:
+        accepted = False
+        reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}")
+    else:
+        reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}")
+
+    if accepted:
+        reasons.append("accept: SDK aggregate gate passed")
+
+    return GateDecision(
+        candidate_id=candidate_id,
+        accepted=accepted,
+        reasons=reasons,
+        train_score_delta=0.0,
+        validation_score_delta=improvement,
+        new_hard_failures=[],
+        protected_regressions=[],
+        validation_new_failures=[],
+        excessive_score_drops=[],
+        overfit_detected=False,
+        candidate_cost=total_cost,
+        cumulative_cost=0.0,
+        total_run_cost=total_cost,
+        cost=total_cost,
+        gate_status="partial_applied",
+        gate_not_applied_reason="SDK OptimizeResult exposes aggregate scores but not full per-case deltas",
+        not_applied_checks=[
+            "per_case_delta",
+            "protected_regression",
+            "new_hard_failure",
+            "max_score_drop_per_case",
+        ],
+    )
+
+
+def _sdk_gate_config(optimizer_config_dict: dict[str, Any]) -> dict[str, float]:
+    gate_payload = optimizer_config_dict.get("gate")
+    if gate_payload is None:
+        gate_payload = {}
+    if not isinstance(gate_payload, dict):
+        raise ValueError("optimizer config field 'gate' must be an object when present")
+
+    min_improvement = gate_payload.get("min_val_score_improvement", 0.01)
+    max_cost = gate_payload.get("max_total_cost", 1.0)
+    if not isinstance(min_improvement, (int, float)) or min_improvement < 0:
+        raise ValueError("optimizer config field 'gate.min_val_score_improvement' must be a non-negative number")
+    if not isinstance(max_cost, (int, float)) or max_cost < 0:
+        raise ValueError("optimizer config field 'gate.max_total_cost' must be a non-negative number")
+    return {
+        "min_val_score_improvement": float(min_improvement),
+        "max_total_cost": float(max_cost),
+    }
+
+
+def _summary_float(summary: dict[str, Any], key: str, default: float) -> float:
+    value = summary.get(key, default)
+    if value is None:
+        return default
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return default
+
+
 def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--train", default=str(DEFAULT_TRAIN), help="Path to train.evalset.json")