Polish SDK prompt audit handling

Author: YAO-001

examples/optimization/eval_optimize_loop/README.md

@@ -78,6 +78,7 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \
   --target-prompt system_prompt=prompts/system.md \
   --target-prompt router_prompt=prompts/router.md \
   --sdk-call-agent your_package.your_module:call_agent \
+  --run-id local-sdk-smoke \
   --output-dir /tmp/eval-optimize-loop-sdk
 ```
 
@@ -92,6 +93,11 @@ follow the SDK `OptimizeConfigFile` schema. Put wrapper-only gate settings in
 "max_total_cost": 1.0}}`). If `--gate-config` is omitted, the wrapper uses the
 same default aggregate gate values as the fake example.
 
+`--target-prompt name=path` may be repeated. If omitted, SDK mode keeps the old
+single-field behavior and registers `system_prompt=--prompt`. A run can optimize
+only `router_prompt`, only `skill_prompt`, or any set of named fields as long as
+`OptimizeResult.best_prompts` returns every registered field.
+
 Fake mode is the complete per-case closed loop. SDK mode is the real
 `AgentOptimizer` / `TargetPrompt` path with an aggregate wrapper gate. When the
 SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult`
@@ -105,6 +111,12 @@ delta, and per-case score-drop checks are not claimed in SDK mode unless the SDK
 exposes full per-case validation scores; they are listed in
 `not_applied_checks`.
 
+Fake mode uses a deterministic run id (`eval_optimize_loop_seed_<seed>`) so the
+example outputs are byte-stable. SDK mode is append-only by default: the wrapper
+derives `run.run_id` from the SDK result `started_at` when available, otherwise
+from the current UTC timestamp. Pass `--run-id` only when a fixed audit path is
+useful for tests or local smoke runs.
+
 ## Source Prompt Writes
 
 The default is **no source write-back**. The baseline prompt file is not modified
@@ -158,8 +170,11 @@ diffs, and the reproducibility command.
 `report.py` also writes audit artifacts under `output_dir/runs/<run_id>/`:
 
 - `config.snapshot.json`;
-- `input_hashes.json`;
-- `candidate_prompts/<candidate_id>/system_prompt.txt`;
+- `input_hashes.json` with train, validation, optimizer, prompt,
+  `target_prompts.<field>`, and optional `gate_config` hashes;
+- fake mode: `candidate_prompts/<candidate_id>/system_prompt.txt`;
+- SDK mode: `candidate_prompts/<candidate_id>/<field_name>.txt` for every
+  returned `best_prompts` field;
 - `case_results/<candidate_id>_<split>.json`;
 - `prompt_diffs/<candidate_id>.diff`.
 

examples/optimization/eval_optimize_loop/eval_loop/backends.py

@@ -119,13 +119,23 @@ async def optimize_async(
             update_source=self.update_source,
             verbose=0,
         )
-        best_prompt = getattr(result, "best_prompts", {}).get("system_prompt")
-        if not best_prompt:
-            raise ValueError("sdk mode completed but OptimizeResult.best_prompts['system_prompt'] was missing")
+        best_prompts = dict(getattr(result, "best_prompts", {}) or {})
+        if not best_prompts:
+            raise ValueError("sdk mode completed but OptimizeResult.best_prompts was empty")
+        missing_fields = [
+            name
+            for name in target_prompt_paths
+            if not best_prompts.get(name)
+        ]
+        if missing_fields:
+            missing = ", ".join(sorted(missing_fields))
+            raise ValueError(
+                "sdk mode completed but OptimizeResult.best_prompts is missing registered target fields: "
+                f"{missing}"
+            )
         self.last_result = result
         self.last_result_summary = _summarize_sdk_result(result)
         self.last_artifact_dir = str(output_dir)
-        best_prompts = dict(getattr(result, "best_prompts", {}) or {})
         baseline_prompts = _read_prompt_bundle(target_prompt_paths)
         return [
             CandidatePrompt(
@@ -176,6 +186,7 @@ def _summarize_sdk_result(result: Any) -> dict[str, Any]:
         "total_llm_cost": _safe_jsonable(getattr(result, "total_llm_cost", 0.0)),
         "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})),
         "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)),
+        "started_at": _safe_jsonable(getattr(result, "started_at", None)),
         "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)),
         "baseline_prompts": _safe_jsonable(getattr(result, "baseline_prompts", {})),
         "best_prompts": _safe_jsonable(getattr(result, "best_prompts", {})),

examples/optimization/eval_optimize_loop/eval_loop/report.py

@@ -277,7 +277,12 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None
         candidate: CandidatePrompt = record["candidate"]
         candidate_dir = prompt_dir / candidate.candidate_id
         candidate_dir.mkdir(exist_ok=True)
-        (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8")
+        best_prompts = report.audit.get("sdk_result_summary", {}).get("best_prompts", {})
+        if report.run.get("mode") == "sdk" and isinstance(best_prompts, dict) and best_prompts:
+            for field_name, prompt_text in best_prompts.items():
+                (candidate_dir / f"{field_name}.txt").write_text(str(prompt_text), encoding="utf-8")
+        else:
+            (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8")
         (diffs_dir / f"{candidate.candidate_id}.diff").write_text(candidate.prompt_diff, encoding="utf-8")
         for split_name in ("train_result", "validation_result"):
             split_result = record[split_name]

examples/optimization/eval_optimize_loop/run_pipeline.py

@@ -7,6 +7,8 @@
 import json
 import sys
 import tempfile
+from datetime import datetime
+from datetime import timezone
 from pathlib import Path
 from typing import Any
 
@@ -55,6 +57,7 @@ def run_pipeline(
     update_source: bool = False,
     gate_config_path: str | Path | None = None,
     target_prompts: list[str] | None = None,
+    run_id: str | None = None,
 ) -> OptimizationReport:
     """Run baseline eval, fake optimization, validation gate, and reports."""
 
@@ -101,6 +104,8 @@ def run_pipeline(
             gate_config=wrapper_gate_config,
             gate_config_path=gate_config_path,
             target_prompt_paths=target_prompt_paths,
+            sdk_call_agent=sdk_call_agent,
+            run_id=run_id,
         )
         write_reports(report, output_dir)
         return report
@@ -334,6 +339,8 @@ def _build_sdk_report(
     gate_config: dict[str, float],
     gate_config_path: str | Path | None,
     target_prompt_paths: dict[str, str | Path],
+    sdk_call_agent: str | None,
+    run_id: str | None,
 ) -> OptimizationReport:
     input_hashes = _input_hashes(
         train_path=train_path,
@@ -351,6 +358,14 @@ def _build_sdk_report(
     )
     total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0)
     duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0)
+    effective_run_id = run_id or _default_sdk_run_id(sdk_summary)
+    target_prompt_hashes = {
+        name: sha256_file(path)
+        for name, path in target_prompt_paths.items()
+    }
+    input_hashes["target_prompts"] = target_prompt_hashes
+    if gate_config_path:
+        input_hashes["gate_config"] = sha256_file(gate_config_path)
     availability = {
         "aggregate_validation_result": True,
         "full_train_eval_result": False,
@@ -401,10 +416,6 @@ def _build_sdk_report(
         for candidate in candidates
     }
     field_prompt_hashes = _candidate_prompt_hashes_by_field(candidates, sdk_summary)
-    target_prompt_hashes = {
-        name: sha256_file(path)
-        for name, path in target_prompt_paths.items()
-    }
     audit = {
         "seed": None,
         "duration_seconds": duration_seconds,
@@ -450,10 +461,12 @@ def _build_sdk_report(
             update_source=update_source,
             gate_config_path=gate_config_path,
             target_prompt_paths=target_prompt_paths,
+            sdk_call_agent=sdk_call_agent,
+            run_id=effective_run_id,
         ),
     }
     run = {
-        "run_id": "eval_optimize_loop_sdk",
+        "run_id": effective_run_id,
         "mode": "sdk",
         "fake_model": False,
         "fake_judge": False,
@@ -507,18 +520,21 @@ def _sdk_reproducibility_command(
     update_source: bool,
     gate_config_path: str | Path | None,
     target_prompt_paths: dict[str, str | Path],
+    sdk_call_agent: str | None,
+    run_id: str,
 ) -> str:
     command = (
         "python examples/optimization/eval_optimize_loop/run_pipeline.py "
         f"--mode sdk --train {train_path} --val {val_path} "
         f"--optimizer-config {optimizer_config_path} --prompt {prompt_path} "
-        f"--output-dir {output_dir} --sdk-call-agent module:function"
+        f"--output-dir {output_dir} --sdk-call-agent {sdk_call_agent or ''}"
     )
     for name, path in target_prompt_paths.items():
         if not (name == "system_prompt" and Path(path) == Path(prompt_path) and len(target_prompt_paths) == 1):
             command += f" --target-prompt {name}={path}"
     if gate_config_path:
         command += f" --gate-config {gate_config_path}"
+    command += f" --run-id {run_id}"
     if update_source:
         command += " --update-source"
     return command
@@ -624,6 +640,21 @@ def _summary_float(summary: dict[str, Any], key: str, default: float) -> float:
         return default
 
 
+def _default_sdk_run_id(sdk_summary: dict[str, Any]) -> str:
+    started_at = sdk_summary.get("started_at")
+    if isinstance(started_at, str) and started_at.strip():
+        source = started_at.strip()
+    else:
+        source = datetime.now(timezone.utc).isoformat(timespec="seconds")
+    safe = []
+    for char in source:
+        if char.isalnum() or char in {"-", "_"}:
+            safe.append(char)
+        else:
+            safe.append("-")
+    return "eval_optimize_loop_sdk_" + "".join(safe).strip("-")
+
+
 def _parse_target_prompt_paths(
     target_prompts: list[str] | None,
     *,
@@ -681,6 +712,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
         action="append",
         help="SDK target prompt path in name=path format. May be repeated. Defaults to system_prompt=--prompt.",
     )
+    parser.add_argument("--run-id", help="Optional report/audit run id. Fake mode keeps its deterministic default.")
     return parser.parse_args(argv)
 
 
@@ -734,6 +766,7 @@ def main(argv: list[str] | None = None) -> OptimizationReport:
         update_source=args.update_source,
         gate_config_path=args.gate_config,
         target_prompts=args.target_prompt,
+        run_id=args.run_id,
     )
     output_dir = Path(args.output_dir)
     print(f"Wrote {output_dir / 'optimization_report.json'}")