Harden eval optimize audit inputs

Author: YAO-001

examples/optimization/eval_optimize_loop/eval_loop/backends.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import importlib
+import math
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
@@ -170,6 +171,8 @@ def _load_call_agent(path: str):
     call_agent = getattr(module, function_name, None)
     if call_agent is None:
         raise ValueError(f"--sdk-call-agent target {path!r} was not found")
+    if not callable(call_agent):
+        raise ValueError(f"--sdk-call-agent target {path!r} was found but is not callable")
     return call_agent
 
 
@@ -184,13 +187,17 @@ def _has_running_loop() -> bool:
 def _summarize_sdk_result(result: Any) -> dict[str, Any]:
     return {
         "status": _safe_jsonable(getattr(result, "status", None)),
-        "baseline_pass_rate": _safe_jsonable(getattr(result, "baseline_pass_rate", None)),
-        "best_pass_rate": _safe_jsonable(getattr(result, "best_pass_rate", None)),
-        "pass_rate_improvement": _safe_jsonable(getattr(result, "pass_rate_improvement", None)),
+        "baseline_pass_rate": _safe_result_field(
+            "baseline_pass_rate", getattr(result, "baseline_pass_rate", None)
+        ),
+        "best_pass_rate": _safe_result_field("best_pass_rate", getattr(result, "best_pass_rate", None)),
+        "pass_rate_improvement": _safe_result_field(
+            "pass_rate_improvement", getattr(result, "pass_rate_improvement", None)
+        ),
         "baseline_metric_breakdown": _safe_jsonable(getattr(result, "baseline_metric_breakdown", {})),
         "best_metric_breakdown": _safe_jsonable(getattr(result, "best_metric_breakdown", {})),
         "metric_thresholds": _safe_jsonable(getattr(result, "metric_thresholds", {})),
-        "total_llm_cost": _safe_jsonable(getattr(result, "total_llm_cost", 0.0)),
+        "total_llm_cost": _safe_result_field("total_llm_cost", getattr(result, "total_llm_cost", 0.0)),
         "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})),
         "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)),
         "started_at": _safe_jsonable(getattr(result, "started_at", None)),
@@ -211,6 +218,13 @@ def _summarize_sdk_result(result: Any) -> dict[str, Any]:
     }
 
 
+def _safe_result_field(field_name: str, value: Any) -> Any:
+    try:
+        return _safe_jsonable(value)
+    except ValueError as exc:
+        raise ValueError(f"SDK OptimizeResult field {field_name} must be a finite number") from exc
+
+
 def _safe_jsonable(value: Any) -> Any:
     if hasattr(value, "model_dump"):
         return _safe_jsonable(value.model_dump(mode="json"))
@@ -220,7 +234,11 @@ def _safe_jsonable(value: Any) -> Any:
         return {str(key): _safe_jsonable(item) for key, item in value.items()}
     if isinstance(value, (list, tuple)):
         return [_safe_jsonable(item) for item in value]
-    if isinstance(value, (str, int, float, bool)) or value is None:
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise ValueError("value must be a finite number")
+        return value
+    if isinstance(value, (str, int, bool)) or value is None:
         return value
     return repr(value)
 

examples/optimization/eval_optimize_loop/eval_loop/report.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import re
 import shutil
 from pathlib import Path
 from typing import Any
@@ -25,6 +26,7 @@
     "--output-dir /tmp/eval-optimize-loop "
     "--fake-model --fake-judge --trace"
 )
+ARTIFACT_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
 
 
 def compute_case_deltas(
@@ -106,7 +108,7 @@ def write_reports(report: OptimizationReport, output_dir: str | Path) -> tuple[P
 
 
 def report_to_json(report: OptimizationReport) -> str:
-    return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True) + "\n"
+    return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True, allow_nan=False) + "\n"
 
 
 def render_markdown(report: OptimizationReport) -> str:
@@ -248,7 +250,7 @@ def render_markdown(report: OptimizationReport) -> str:
 
 
 def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None:
-    run_id = str(report.run.get("run_id") or "run")
+    run_id = _safe_artifact_name(str(report.run.get("run_id") or "run"))
     run_dir = output_path / "runs" / run_id
     if run_dir.exists() and report.run.get("mode") == "fake":
         shutil.rmtree(run_dir)
@@ -275,20 +277,26 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None
 
     for record in report.candidates:
         candidate: CandidatePrompt = record["candidate"]
-        candidate_dir = prompt_dir / candidate.candidate_id
+        candidate_name = _safe_artifact_name(candidate.candidate_id)
+        candidate_dir = prompt_dir / candidate_name
         candidate_dir.mkdir(exist_ok=True)
         best_prompts = report.audit.get("sdk_result_summary", {}).get("best_prompts", {})
         if report.run.get("mode") == "sdk" and isinstance(best_prompts, dict) and best_prompts:
             for field_name, prompt_text in best_prompts.items():
-                (candidate_dir / f"{field_name}.txt").write_text(str(prompt_text), encoding="utf-8")
+                field_artifact = _safe_artifact_name(str(field_name))
+                (candidate_dir / f"{field_artifact}.txt").write_text(str(prompt_text), encoding="utf-8")
             (candidate_dir / "bundle.txt").write_text(candidate.prompt, encoding="utf-8")
         else:
             (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8")
-        (diffs_dir / f"{candidate.candidate_id}.diff").write_text(candidate.prompt_diff, encoding="utf-8")
+        (diffs_dir / f"{candidate_name}.diff").write_text(candidate.prompt_diff, encoding="utf-8")
         for split_name in ("train_result", "validation_result"):
             split_result = record[split_name]
-            path = results_dir / f"{candidate.candidate_id}_{split_result.split}.json"
-            path.write_text(json.dumps(to_jsonable(split_result), indent=2, sort_keys=True) + "\n", encoding="utf-8")
+            split_artifact = _safe_artifact_name(str(split_result.split))
+            path = results_dir / f"{candidate_name}_{split_artifact}.json"
+            path.write_text(
+                json.dumps(to_jsonable(split_result), indent=2, sort_keys=True, allow_nan=False) + "\n",
+                encoding="utf-8",
+            )
 
 
 def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float) -> str:
@@ -301,3 +309,9 @@ def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float)
     if delta < 0:
         return "score_down"
     return "unchanged"
+
+
+def _safe_artifact_name(name: str) -> str:
+    if name in {"", ".", ".."} or not ARTIFACT_NAME_RE.fullmatch(name):
+        raise ValueError(f"unsafe audit artifact name: {name!r}")
+    return name

examples/optimization/eval_optimize_loop/run_pipeline.py

@@ -6,6 +6,7 @@
 import hashlib
 import json
 import math
+import re
 import shlex
 import sys
 import tempfile
@@ -42,6 +43,8 @@
 DEFAULT_OPTIMIZER_CONFIG = HERE / "data" / "optimizer.json"
 DEFAULT_PROMPT = HERE / "prompts" / "baseline_system_prompt.txt"
 DEFAULT_OUTPUT_DIR = Path(tempfile.gettempdir()) / "eval-optimize-loop"
+TARGET_PROMPT_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+RUN_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
 
 
 def run_pipeline(
@@ -65,6 +68,8 @@ def run_pipeline(
 
     if mode not in {"fake", "sdk"}:
         raise ValueError("field 'mode' must be one of: fake, sdk")
+    if run_id is not None:
+        run_id = validate_run_id(run_id)
     if mode == "fake" and (not fake_model or not fake_judge):
         raise ValueError(
             "fake mode requires fake_model=True and fake_judge=True. Pass --fake-model --fake-judge "
@@ -109,6 +114,8 @@ def run_pipeline(
             sdk_call_agent=sdk_call_agent,
             run_id=run_id,
         )
+        if run_id is None:
+            _resolve_default_sdk_run_id_collision(report, output_dir)
         write_reports(report, output_dir)
         return report
 
@@ -351,14 +358,15 @@ def _build_sdk_report(
         prompt_path=prompt_path,
     )
     sdk_summary = sdk_backend.last_result_summary or {}
-    baseline_pass_rate = _summary_float(sdk_summary, "baseline_pass_rate", 0.0)
-    best_pass_rate = _summary_float(sdk_summary, "best_pass_rate", baseline_pass_rate)
+    baseline_pass_rate = _summary_float(sdk_summary, "baseline_pass_rate", 0.0, required=True)
+    best_pass_rate = _summary_float(sdk_summary, "best_pass_rate", baseline_pass_rate, required=True)
     pass_rate_improvement = _summary_float(
         sdk_summary,
         "pass_rate_improvement",
         best_pass_rate - baseline_pass_rate,
+        required=True,
     )
-    total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0)
+    total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True)
     duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0)
     effective_run_id = run_id or _default_sdk_run_id(sdk_summary)
     target_prompt_hashes = {
@@ -562,8 +570,8 @@ def _sdk_gate_decision(
     gate_config: dict[str, float],
 ) -> GateDecision:
     status = str(sdk_summary.get("status") or "UNKNOWN")
-    improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0)
-    total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0)
+    improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0, required=True)
+    total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True)
     min_improvement = gate_config["min_val_score_improvement"]
     max_cost = gate_config["max_total_cost"]
 
@@ -659,14 +667,25 @@ def _is_non_negative_finite_number(value: Any) -> bool:
     )
 
 
-def _summary_float(summary: dict[str, Any], key: str, default: float) -> float:
+def _summary_float(summary: dict[str, Any], key: str, default: float, *, required: bool = False) -> float:
     value = summary.get(key, default)
     if value is None:
         return default
+    if isinstance(value, bool):
+        if required:
+            raise ValueError(f"SDK OptimizeResult field {key} must be a finite number")
+        return default
     try:
-        return float(value)
+        parsed = float(value)
     except (TypeError, ValueError):
+        if required:
+            raise ValueError(f"SDK OptimizeResult field {key} must be a finite number")
+        return default
+    if not math.isfinite(parsed):
+        if required:
+            raise ValueError(f"SDK OptimizeResult field {key} must be a finite number")
         return default
+    return parsed
 
 
 def _default_sdk_run_id(sdk_summary: dict[str, Any]) -> str:
@@ -704,16 +723,39 @@ def _parse_target_prompt_paths(
         if "=" not in item:
             raise ValueError("--target-prompt must use name=path format")
         name, path = item.split("=", 1)
-        name = name.strip()
         path = path.strip()
-        if not name or not path:
+        if not TARGET_PROMPT_FIELD_RE.fullmatch(name):
+            raise ValueError(
+                f"--target-prompt field name {name!r} is invalid; use /^[A-Za-z_][A-Za-z0-9_]*$/"
+            )
+        if not path:
             raise ValueError("--target-prompt must use non-empty name=path values")
         if name in parsed:
             raise ValueError(f"--target-prompt duplicate field name {name!r}")
         parsed[name] = Path(path)
     return parsed
 
 
+def validate_run_id(run_id: str) -> str:
+    if not isinstance(run_id, str):
+        raise ValueError(f"--run-id value {run_id!r} must be a string")
+    if run_id in {"", ".", ".."} or not RUN_ID_RE.fullmatch(run_id):
+        raise ValueError(f"--run-id value {run_id!r} is invalid")
+    return run_id
+
+
+def _resolve_default_sdk_run_id_collision(report: OptimizationReport, output_dir: str | Path) -> None:
+    base_run_id = str(report.run.get("run_id") or "")
+    validate_run_id(base_run_id)
+    run_root = Path(output_dir) / "runs"
+    candidate = base_run_id
+    suffix = 1
+    while (run_root / candidate).exists():
+        candidate = f"{base_run_id}-{suffix}"
+        suffix += 1
+    report.run["run_id"] = candidate
+
+
 def _candidate_prompt_hashes_by_field(
     candidates: list[CandidatePrompt],
     sdk_summary: dict[str, Any],

examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py

@@ -10,6 +10,7 @@
 from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN
 from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL
 from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline
+from examples.optimization.eval_optimize_loop.eval_loop.report import report_to_json
 
 
 def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path):
@@ -69,6 +70,7 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path):
         "total_run_cost",
     }
     assert payload["run"]["reproducibility_command"].startswith("python examples/optimization")
+    assert payload["run"]["run_id"] == "eval_optimize_loop_seed_91"
     assert payload["audit"]["total_run_cost"] == payload["audit"]["cost"]["total"]
     assert "candidate_001_overfit" in payload["audit"]["prompt_diffs"]
     assert "candidate_002_safe" in payload["audit"]["prompt_diffs"]
@@ -157,3 +159,24 @@ def test_cli_mode_fake_and_legacy_fake_flags_both_run(tmp_path: Path):
 
     assert (first / "optimization_report.json").is_file()
     assert (second / "optimization_report.md").is_file()
+
+
+def test_report_to_json_rejects_nan_values(tmp_path: Path):
+    report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True)
+    report.audit["bad_float"] = float("nan")
+
+    try:
+        report_to_json(report)
+    except ValueError as exc:
+        assert "Out of range float values are not JSON compliant" in str(exc)
+    else:
+        raise AssertionError("report_to_json should reject NaN")
+
+
+def test_fake_report_json_remains_strict_json(tmp_path: Path):
+    report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True)
+    payload = report_to_json(report)
+
+    assert "NaN" not in payload
+    assert "Infinity" not in payload
+    assert json.loads(payload)["selected_candidate"] == "candidate_002_safe"