Harden SDK audit reproducibility

Author: YAO-001

examples/optimization/eval_optimize_loop/README.md

@@ -84,7 +84,10 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \
 
 `--sdk-call-agent` must point to an async callable compatible with
 `AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials
-needed by that callable. SDK mode never silently falls back to fake mode.
+needed by that callable. SDK mode never silently falls back to fake mode. The
+generated reproducibility command records the actual `--sdk-call-agent`
+`module:function` target and file/config paths, but it does not record API keys
+or other provider secrets.
 
 SDK optimizer config and wrapper gate config are intentionally separate.
 `--optimizer-config` is passed unchanged to `AgentOptimizer.optimize`, so it must
@@ -113,9 +116,10 @@ exposes full per-case validation scores; they are listed in
 
 Fake mode uses a deterministic run id (`eval_optimize_loop_seed_<seed>`) so the
 example outputs are byte-stable. SDK mode is append-only by default: the wrapper
-derives `run.run_id` from the SDK result `started_at` when available, otherwise
-from the current UTC timestamp. Pass `--run-id` only when a fixed audit path is
-useful for tests or local smoke runs.
+derives a compact UTC `run.run_id` from the SDK result `started_at` when
+available, otherwise from the current UTC timestamp. Pass `--run-id` only when
+a fixed audit path is useful for tests or local smoke runs; only explicit
+`--run-id` values are included in the reproducibility command.
 
 ## Source Prompt Writes
 
@@ -174,7 +178,8 @@ diffs, and the reproducibility command.
   `target_prompts.<field>`, and optional `gate_config` hashes;
 - fake mode: `candidate_prompts/<candidate_id>/system_prompt.txt`;
 - SDK mode: `candidate_prompts/<candidate_id>/<field_name>.txt` for every
-  returned `best_prompts` field;
+  returned `best_prompts` field, plus `bundle.txt` with the combined prompt
+  shown in the wrapper report;
 - `case_results/<candidate_id>_<split>.json`;
 - `prompt_diffs/<candidate_id>.diff`.
 

examples/optimization/eval_optimize_loop/eval_loop/backends.py

@@ -122,17 +122,24 @@ async def optimize_async(
         best_prompts = dict(getattr(result, "best_prompts", {}) or {})
         if not best_prompts:
             raise ValueError("sdk mode completed but OptimizeResult.best_prompts was empty")
-        missing_fields = [
-            name
-            for name in target_prompt_paths
-            if not best_prompts.get(name)
-        ]
+        missing_fields = [name for name in target_prompt_paths if name not in best_prompts]
         if missing_fields:
             missing = ", ".join(sorted(missing_fields))
             raise ValueError(
                 "sdk mode completed but OptimizeResult.best_prompts is missing registered target fields: "
                 f"{missing}"
             )
+        empty_fields = [
+            name
+            for name in target_prompt_paths
+            if not isinstance(best_prompts[name], str) or not best_prompts[name].strip()
+        ]
+        if empty_fields:
+            empty = ", ".join(sorted(empty_fields))
+            raise ValueError(
+                "sdk mode completed but OptimizeResult.best_prompts contained empty registered target fields: "
+                f"{empty}"
+            )
         self.last_result = result
         self.last_result_summary = _summarize_sdk_result(result)
         self.last_artifact_dir = str(output_dir)

examples/optimization/eval_optimize_loop/eval_loop/report.py

@@ -281,6 +281,7 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None
         if report.run.get("mode") == "sdk" and isinstance(best_prompts, dict) and best_prompts:
             for field_name, prompt_text in best_prompts.items():
                 (candidate_dir / f"{field_name}.txt").write_text(str(prompt_text), encoding="utf-8")
+            (candidate_dir / "bundle.txt").write_text(candidate.prompt, encoding="utf-8")
         else:
             (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8")
         (diffs_dir / f"{candidate.candidate_id}.diff").write_text(candidate.prompt_diff, encoding="utf-8")

examples/optimization/eval_optimize_loop/run_pipeline.py

@@ -5,6 +5,8 @@
 import argparse
 import hashlib
 import json
+import math
+import shlex
 import sys
 import tempfile
 from datetime import datetime
@@ -462,7 +464,7 @@ def _build_sdk_report(
             gate_config_path=gate_config_path,
             target_prompt_paths=target_prompt_paths,
             sdk_call_agent=sdk_call_agent,
-            run_id=effective_run_id,
+            run_id=run_id,
         ),
     }
     run = {
@@ -521,23 +523,36 @@ def _sdk_reproducibility_command(
     gate_config_path: str | Path | None,
     target_prompt_paths: dict[str, str | Path],
     sdk_call_agent: str | None,
-    run_id: str,
+    run_id: str | None,
 ) -> str:
-    command = (
-        "python examples/optimization/eval_optimize_loop/run_pipeline.py "
-        f"--mode sdk --train {train_path} --val {val_path} "
-        f"--optimizer-config {optimizer_config_path} --prompt {prompt_path} "
-        f"--output-dir {output_dir} --sdk-call-agent {sdk_call_agent or ''}"
-    )
+    parts = [
+        "python",
+        "examples/optimization/eval_optimize_loop/run_pipeline.py",
+        "--mode",
+        "sdk",
+        "--train",
+        str(train_path),
+        "--val",
+        str(val_path),
+        "--optimizer-config",
+        str(optimizer_config_path),
+        "--prompt",
+        str(prompt_path),
+        "--output-dir",
+        str(output_dir),
+        "--sdk-call-agent",
+        sdk_call_agent or "",
+    ]
     for name, path in target_prompt_paths.items():
         if not (name == "system_prompt" and Path(path) == Path(prompt_path) and len(target_prompt_paths) == 1):
-            command += f" --target-prompt {name}={path}"
+            parts.extend(["--target-prompt", f"{name}={path}"])
     if gate_config_path:
-        command += f" --gate-config {gate_config_path}"
-    command += f" --run-id {run_id}"
+        parts.extend(["--gate-config", str(gate_config_path)])
+    if run_id:
+        parts.extend(["--run-id", run_id])
     if update_source:
-        command += " --update-source"
-    return command
+        parts.append("--update-source")
+    return " ".join(shlex.quote(part) for part in parts)
 
 
 def _sdk_gate_decision(
@@ -620,16 +635,30 @@ def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, floa
 
     min_improvement = gate_payload.get("min_val_score_improvement", 0.01)
     max_cost = gate_payload.get("max_total_cost", 1.0)
-    if not isinstance(min_improvement, (int, float)) or min_improvement < 0:
-        raise ValueError(f"{path_text}: field 'gate.min_val_score_improvement' must be a non-negative number")
-    if not isinstance(max_cost, (int, float)) or max_cost < 0:
-        raise ValueError(f"{path_text}: field 'gate.max_total_cost' must be a non-negative number")
+    if not _is_non_negative_finite_number(min_improvement):
+        raise ValueError(
+            f"--gate-config {path_text}: field 'gate.min_val_score_improvement' "
+            "must be a non-negative finite number"
+        )
+    if not _is_non_negative_finite_number(max_cost):
+        raise ValueError(
+            f"--gate-config {path_text}: field 'gate.max_total_cost' must be a non-negative finite number"
+        )
     return {
         "min_val_score_improvement": float(min_improvement),
         "max_total_cost": float(max_cost),
     }
 
 
+def _is_non_negative_finite_number(value: Any) -> bool:
+    return (
+        isinstance(value, (int, float))
+        and not isinstance(value, bool)
+        and math.isfinite(float(value))
+        and float(value) >= 0
+    )
+
+
 def _summary_float(summary: dict[str, Any], key: str, default: float) -> float:
     value = summary.get(key, default)
     if value is None:
@@ -644,8 +673,16 @@ def _default_sdk_run_id(sdk_summary: dict[str, Any]) -> str:
     started_at = sdk_summary.get("started_at")
     if isinstance(started_at, str) and started_at.strip():
         source = started_at.strip()
+        try:
+            normalized = source[:-1] + "+00:00" if source.endswith("Z") else source
+            parsed = datetime.fromisoformat(normalized)
+            if parsed.tzinfo is None:
+                parsed = parsed.replace(tzinfo=timezone.utc)
+            return "eval_optimize_loop_sdk_" + parsed.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        except ValueError:
+            pass
     else:
-        source = datetime.now(timezone.utc).isoformat(timespec="seconds")
+        return "eval_optimize_loop_sdk_" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
     safe = []
     for char in source:
         if char.isalnum() or char in {"-", "_"}:

examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py

@@ -139,6 +139,22 @@ def test_sdk_backend_missing_registered_best_prompt_field_is_clear(tmp_path: Pat
         )
 
 
+def test_sdk_backend_empty_best_prompts_dict_error_is_clear(tmp_path: Path, monkeypatch):
+    _install_fake_sdk(monkeypatch, best_prompts={})
+    prompt_path = tmp_path / "prompt.txt"
+    prompt_path.write_text("baseline", encoding="utf-8")
+    backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent")
+
+    with pytest.raises(ValueError, match="best_prompts was empty"):
+        backend.optimize(
+            baseline_prompt="baseline",
+            train_path=tmp_path / "train.evalset.json",
+            val_path=tmp_path / "val.evalset.json",
+            optimizer_config_path=tmp_path / "optimizer.json",
+            output_dir=tmp_path / "out",
+        )
+
+
 def test_sdk_backend_passes_update_source_true(tmp_path: Path, monkeypatch):
     calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt")
     prompt_path = tmp_path / "prompt.txt"
@@ -207,7 +223,7 @@ def test_sdk_backend_empty_best_prompt_error_is_clear(tmp_path: Path, monkeypatc
     prompt_path.write_text("baseline", encoding="utf-8")
     backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent")
 
-    with pytest.raises(ValueError, match="missing registered target fields.*system_prompt"):
+    with pytest.raises(ValueError, match="contained empty registered target fields.*system_prompt"):
         backend.optimize(
             baseline_prompt="baseline",
             train_path=tmp_path / "train.evalset.json",
@@ -359,8 +375,9 @@ def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path
         sdk_call_agent="fake_call_agent_module:call_agent",
     )
 
-    assert report.run["run_id"] == "eval_optimize_loop_sdk_2026-07-04T12-34-56-00-00"
+    assert report.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z"
     assert (tmp_path / "sdk_run" / "runs" / report.run["run_id"]).is_dir()
+    assert "--run-id" not in report.run["reproducibility_command"]
 
 
 def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_gate(
@@ -457,6 +474,39 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Pa
     assert any("cost" in reason for reason in decision.reasons)
 
 
+@pytest.mark.parametrize(
+    ("field_name", "field_value"),
+    [
+        ("min_val_score_improvement", True),
+        ("max_total_cost", float("nan")),
+        ("max_total_cost", float("inf")),
+    ],
+)
+def test_run_pipeline_mode_sdk_rejects_invalid_gate_numbers(
+    tmp_path: Path,
+    monkeypatch,
+    field_name,
+    field_value,
+):
+    _install_fake_sdk(monkeypatch, best_prompt="optimized prompt")
+    gate = {"min_val_score_improvement": 0.01, "max_total_cost": 1.0}
+    gate[field_name] = field_value
+    gate_path = tmp_path / "bad_gate.json"
+    gate_path.write_text(json.dumps({"gate": gate}), encoding="utf-8")
+
+    with pytest.raises(ValueError, match=f"--gate-config.*{field_name}"):
+        run_pipeline(
+            mode="sdk",
+            train_path=DEFAULT_TRAIN,
+            val_path=DEFAULT_VAL,
+            optimizer_config_path=_write_sdk_optimizer_config(tmp_path),
+            prompt_path=DEFAULT_PROMPT,
+            output_dir=tmp_path / "sdk_run",
+            sdk_call_agent="fake_call_agent_module:call_agent",
+            gate_config_path=gate_path,
+        )
+
+
 def test_run_pipeline_mode_sdk_does_not_pass_wrapper_gate_config_to_agent_optimizer(
     tmp_path: Path,
     monkeypatch,
@@ -540,12 +590,16 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path:
     assert (run_dir / "candidate_prompts" / "sdk_best" / "skill_prompt.txt").read_text(
         encoding="utf-8"
     ) == "optimized skill"
+    assert (run_dir / "candidate_prompts" / "sdk_best" / "bundle.txt").read_text(
+        encoding="utf-8"
+    ) == report.candidates[0]["candidate"].prompt
     input_hashes = json.loads((run_dir / "input_hashes.json").read_text(encoding="utf-8"))
     assert set(input_hashes["target_prompts"]) == {"system_prompt", "router_prompt", "skill_prompt"}
     assert "gate_config" in input_hashes
     command = report.run["reproducibility_command"]
     assert "--sdk-call-agent fake_call_agent_module:call_agent" in command
-    assert f"--target-prompt router_prompt={router_path}" in command
+    assert "--target-prompt" in command
+    assert f"router_prompt={router_path}" in command
     assert "--gate-config" in command