From 88baed0f3de85eae85a27f3019a3b45f30065d61 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 18:07:03 +0800 Subject: [PATCH 01/34] Add eval optimize loop example --- .gitignore | 2 + .../optimization/eval_optimize_loop/DESIGN.md | 12 + .../optimization/eval_optimize_loop/README.md | 131 +++ .../eval_optimize_loop/data/optimizer.json | 16 + .../data/train.evalset.json | 57 ++ .../eval_optimize_loop/data/val.evalset.json | 60 ++ .../eval_optimize_loop/eval_loop/__init__.py | 24 + .../eval_loop/attribution.py | 64 ++ .../eval_optimize_loop/eval_loop/evaluator.py | 74 ++ .../eval_loop/fake_judge.py | 132 +++ .../eval_loop/fake_model.py | 99 ++ .../eval_optimize_loop/eval_loop/gate.py | 106 +++ .../eval_optimize_loop/eval_loop/loader.py | 45 + .../eval_optimize_loop/eval_loop/optimizer.py | 52 ++ .../eval_optimize_loop/eval_loop/report.py | 198 ++++ .../eval_optimize_loop/eval_loop/schemas.py | 139 +++ .../eval_optimize_loop/eval_loop/trace.py | 18 + .../outputs/optimization_report.example.json | 845 ++++++++++++++++++ .../outputs/optimization_report.example.md | 74 ++ .../prompts/baseline_system_prompt.txt | 4 + .../eval_optimize_loop/run_pipeline.py | 253 ++++++ .../tests/test_failure_attribution.py | 47 + .../eval_optimize_loop/tests/test_gate.py | 108 +++ .../tests/test_pipeline_fake_mode.py | 63 ++ pyproject.toml | 2 +- 25 files changed, 2624 insertions(+), 1 deletion(-) create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/data/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/data/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/val.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/attribution.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/evaluator.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/fake_model.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/gate.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/loader.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/optimizer.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/report.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/schemas.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/trace.py create mode 100644 examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json create mode 100644 examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md create mode 100644 examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt create mode 100644 examples/optimization/eval_optimize_loop/run_pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_gate.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py diff --git a/.gitignore b/.gitignore index 233248dd..eb3054c4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ *.lock *.log examples/*.log +examples/optimization/eval_optimize_loop/outputs/optimization_report.json +examples/optimization/eval_optimize_loop/outputs/optimization_report.md trpc-agent-py.egg-info diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..7208ec16 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,12 @@ +# 设计说明 + +本示例把失败归因、候选生成、验证门禁和报告持久化拆成独立模块，便于审阅。 +失败归因采用规则表：JSON 解析失败归为格式违规，缺字段和值不匹配归为最终答案不一致， +rubric 缺项归为裁判失败，超长归为长度违规；每个失败用例都保留原因和证据。 +门禁先比较 train/validation 的总体变化，再检查新增硬失败、受保护用例回退、 +单用例最大降分和成本预算，避免只在训练集变好的候选被接受。过拟合候选会强制 JSON， +故能提升训练集却伤害自然语言验证用例和受保护 yes/no 用例；安全候选只在用户明确要求时启用严格格式。 +报告同时写 JSON 和 Markdown，记录 seed、成本、耗时、配置哈希、候选 prompt 与 diff，形成可追溯审计。 +fake mode 的目的不是模拟真实模型能力，而是让评审和 CI 在无 API key 条件下稳定复现闭环行为。 +这种设计也降低了迁移成本：真实接入时只需替换模型调用和优化器，评估结果、门禁策略与报告结构仍可沿用， +从而把示例中的确定性保障保留下来。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..5b8a0d7e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,131 @@ +# Evaluation + Optimization Closed Loop + +This example implements a reproducible evaluation and prompt-optimization loop +that runs without `TRPC_AGENT_API_KEY` or any external model provider. It is an +example-local implementation: the code mirrors the evaluator/optimizer workflow +with a thin adapter so the behavior is deterministic and easy to review. + +## Architecture + +```text +train.evalset.json val.evalset.json baseline_system_prompt.txt + | | | + v v v + loader.py --------------> evaluator.py <----------- fake_model.py + | | | + | v | + | fake_judge.py | + | | | + v v | + attribution.py <------ baseline + candidate results | + | | | + v v | + optimizer.py --------> candidate prompts --------------+ + | | + v v + gate.py ------------> accept/reject decisions + | + v + report.py ----------> optimization_report.json/.md +``` + +## Quick Start + +Short deterministic command: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --fake-model --fake-judge --trace +``` + +Full command with explicit inputs: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --train examples/optimization/eval_optimize_loop/data/train.evalset.json \ + --val examples/optimization/eval_optimize_loop/data/val.evalset.json \ + --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json \ + --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt \ + --output-dir /tmp/eval-optimize-loop \ + --fake-model \ + --fake-judge \ + --trace +``` + +The default output directory is the system temp directory, +`eval-optimize-loop`. The fixed checked-in examples are: + +- `outputs/optimization_report.example.json` +- `outputs/optimization_report.example.md` + +## What The Example Demonstrates + +- Baseline evaluation on train and validation evalsets. +- Rule-based failure attribution for failed cases. +- A fake optimizer that proposes two candidates: + - `candidate_001_overfit`: improves train score but regresses validation. + - `candidate_002_safe`: improves validation without protected regression. +- Candidate validation on the full validation set. +- A configurable gate that rejects train-only gains and protected-case + regressions. +- Audit persistence with seed, cost, deterministic duration, config hash, and + candidate prompt text. + +The six eval cases cover: + +- optimization succeeds: `candidate_002_safe` is accepted; +- optimization has no effect: rubric cases stay unchanged; +- optimization regresses/overfits: `candidate_001_overfit` forces JSON on + validation prose and protected exact-answer cases. + +## Fake Model, Fake Judge, And Trace Mode + +`--fake-model` uses deterministic prompt markers to simulate baseline, overfit, +and safe candidate behavior. `--fake-judge` scores JSON, exact-answer, and +rubric cases locally. `--trace` stores per-case fake model and judge decisions +inside `optimization_report.json`, which makes failures reviewable without +calling an external service. + +This example currently requires `--fake-model --fake-judge`. That is deliberate: +it keeps the PR deterministic and ensures the example runs in CI without API +keys. + +## Inspecting Reports + +Open `optimization_report.md` first for the human-readable decision: + +- final selected candidate; +- why the overfit candidate was rejected; +- why the safe candidate was accepted; +- baseline vs candidate score table; +- per-case delta table; +- failure attribution summary; +- prompt diffs; +- reproducibility command. + +Use `optimization_report.json` for automation and audit. It includes the same +decision data plus full case outputs, traces, costs, config hash, and candidate +prompt snapshots. + +## Run Tests + +```bash +python -m pytest examples/optimization/eval_optimize_loop/tests +``` + +The tests cover gate rejection/acceptance, failure attribution, fake-mode report +generation, and deterministic output with the same seed. + +## Switching To Real Evaluator/Optimizer Later + +The example intentionally isolates integration points: + +- replace `ExampleEvaluator` in `eval_loop/evaluator.py` with an adapter around + `AgentEvaluator` when real agent execution and SDK evalsets are desired; +- replace `FakeOptimizer` in `eval_loop/optimizer.py` with `AgentOptimizer` or a + remote optimizer; +- keep `gate.py`, `attribution.py`, and `report.py` as reviewable policy and + audit layers around the real components. + +When switching to real mode, keep train and validation files distinct, preserve +the protected-case gate, and continue writing both JSON and Markdown reports so +optimization decisions remain reviewable. diff --git a/examples/optimization/eval_optimize_loop/data/optimizer.json b/examples/optimization/eval_optimize_loop/data/optimizer.json new file mode 100644 index 00000000..e7904006 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/optimizer.json @@ -0,0 +1,16 @@ +{ + "seed": 91, + "optimizer": { + "name": "fake_two_candidate_optimizer", + "description": "Deterministically proposes one overfit candidate and one safe candidate." + }, + "gate": { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": false, + "protected_case_ids": [ + "val_protected_yes_no" + ], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0 + } +} diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 00000000..d2b3c544 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,57 @@ +{ + "evalset_id": "eval_optimize_loop_train_v1", + "split": "train", + "description": "Training cases used for failure attribution and fake prompt optimization.", + "cases": [ + { + "id": "train_json_refund", + "input": "Return strict JSON for a refund escalation. Use keys intent and priority.", + "expectation": { + "type": "json", + "required_keys": [ + "intent", + "priority" + ], + "expected_values": { + "intent": "refund", + "priority": "high" + } + }, + "tags": [ + "json", + "format" + ] + }, + { + "id": "train_exact_order_status", + "input": "Answer exactly READY when the order can ship.", + "expectation": { + "type": "exact", + "expected": "READY" + }, + "tags": [ + "exact", + "format" + ] + }, + { + "id": "train_rubric_retry_summary", + "input": "In 80 chars or less, mention latency and retries.", + "expectation": { + "type": "rubric", + "must_include": [ + "latency", + "retries" + ], + "forbidden": [ + "database" + ], + "max_chars": 80 + }, + "tags": [ + "rubric", + "no_effect" + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 00000000..a993ea98 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,60 @@ +{ + "evalset_id": "eval_optimize_loop_validation_v1", + "split": "validation", + "description": "Validation cases that separate safe optimization from train-set overfitting.", + "cases": [ + { + "id": "val_json_invoice", + "input": "Return strict JSON for an invoice approval. Use keys status and next_step.", + "expectation": { + "type": "json", + "required_keys": [ + "status", + "next_step" + ], + "expected_values": { + "next_step": "email_customer", + "status": "approved" + } + }, + "tags": [ + "json", + "safe_improves" + ] + }, + { + "id": "val_explain_cache", + "input": "Explain cache invalidation for a junior engineer in natural language.", + "expectation": { + "type": "rubric", + "must_include": [ + "cache", + "stale data" + ], + "forbidden": [ + "{", + "}", + "json" + ], + "max_chars": 180 + }, + "tags": [ + "rubric", + "overfit_guard" + ] + }, + { + "id": "val_protected_yes_no", + "input": "Answer exactly YES if idempotent retries are safe for duplicate requests.", + "expectation": { + "type": "exact", + "expected": "YES" + }, + "protected": true, + "tags": [ + "protected", + "exact" + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/eval_loop/__init__.py b/examples/optimization/eval_optimize_loop/eval_loop/__init__.py new file mode 100644 index 00000000..74835adb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/__init__.py @@ -0,0 +1,24 @@ +"""Deterministic evaluation + optimization loop example. + +This package is intentionally example-local. It mirrors the shape of an +Evaluator/Optimizer workflow while keeping fake model and fake judge execution +offline, deterministic, and easy to inspect. +""" + +from .schemas import CandidatePrompt +from .schemas import CaseDelta +from .schemas import CaseResult +from .schemas import EvalCase +from .schemas import EvalResult +from .schemas import GateDecision +from .schemas import OptimizationReport + +__all__ = [ + "CandidatePrompt", + "CaseDelta", + "CaseResult", + "EvalCase", + "EvalResult", + "GateDecision", + "OptimizationReport", +] diff --git a/examples/optimization/eval_optimize_loop/eval_loop/attribution.py b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py new file mode 100644 index 00000000..246185ff --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py @@ -0,0 +1,64 @@ +"""Rule-based failure attribution for the example evaluator.""" + +from __future__ import annotations + +from collections import Counter +from typing import Iterable + +from .schemas import EvalResult + + +_ERROR_TO_ATTRIBUTION = { + "json_parse_failure": ("format_violation", "output is not valid JSON"), + "required_key_missing": ("final_answer_mismatch", "required JSON key is missing"), + "json_value_mismatch": ("final_answer_mismatch", "JSON value does not match expected value"), + "exact_answer_mismatch": ("final_answer_mismatch", "normalized exact answer mismatch"), + "forbidden_pattern": ("format_violation", "output contains a forbidden pattern"), + "missing_rubric_terms": ("llm_rubric_failed", "required rubric terms are missing"), + "max_chars_exceeded": ("length_violation", "output exceeds max_chars"), +} + + +def attribute_failure(error_code: str, evidence: str) -> tuple[str, str, str]: + """Return failure_category, failure_reason, evidence for a judge error.""" + + category, reason = _ERROR_TO_ATTRIBUTION.get( + error_code, + ("unknown_failure", f"unmapped judge error: {error_code}"), + ) + return category, reason, evidence + + +def summarize_failures(results: Iterable[EvalResult]) -> dict[str, object]: + """Summarize failures by category and by prompt/split for reporting.""" + + by_category: Counter[str] = Counter() + by_prompt: dict[str, dict[str, int]] = {} + examples: list[dict[str, str]] = [] + total_failed = 0 + + for result in results: + prompt_key = f"{result.prompt_id}:{result.split}" + by_prompt.setdefault(prompt_key, {}) + for case in result.cases: + if case.passed: + continue + total_failed += 1 + category = case.failure_category or "unknown_failure" + by_category[category] += 1 + by_prompt[prompt_key][category] = by_prompt[prompt_key].get(category, 0) + 1 + examples.append({ + "prompt_id": result.prompt_id, + "split": result.split, + "case_id": case.case_id, + "failure_category": category, + "failure_reason": case.failure_reason or "", + "evidence": case.evidence or "", + }) + + return { + "total_failed_cases": total_failed, + "by_category": dict(sorted(by_category.items())), + "by_prompt_split": {key: dict(sorted(value.items())) for key, value in sorted(by_prompt.items())}, + "examples": examples, + } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py new file mode 100644 index 00000000..4bd1a4c9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py @@ -0,0 +1,74 @@ +"""Example-local evaluator adapter. + +This adapter intentionally keeps the surface small: it drives a model callable, +uses the fake judge for deterministic scoring, and returns dataclass results. +It can later be replaced with SDK AgentEvaluator integration. +""" + +from __future__ import annotations + +from typing import Iterable + +from .attribution import attribute_failure +from .fake_judge import FakeJudge +from .fake_model import FakeModel +from .schemas import CaseResult +from .schemas import EvalCase +from .schemas import EvalResult +from .trace import make_trace + + +class ExampleEvaluator: + """Evaluate prompt text against cases with fake model/judge components.""" + + def __init__(self, model: FakeModel, judge: FakeJudge, *, trace_enabled: bool = False) -> None: + self.model = model + self.judge = judge + self.trace_enabled = trace_enabled + + def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: + case_results: list[CaseResult] = [] + for case in cases: + output, model_trace, cost = self.model.generate(prompt_id, prompt, case) + judged = self.judge.score(case, output) + failure_category = None + failure_reason = None + evidence = None + if not judged.passed: + failure_category, failure_reason, evidence = attribute_failure( + judged.error_code or "unknown_failure", + judged.evidence or "", + ) + trace = make_trace( + self.trace_enabled, + prompt_id=prompt_id, + case_id=case.case_id, + model_trace=model_trace, + judge_trace=judged.trace or {}, + ) + case_results.append( + CaseResult( + case_id=case.case_id, + split=case.split, + score=round(judged.score, 6), + passed=judged.passed, + output=output, + trace=trace, + failure_category=failure_category, + failure_reason=failure_reason, + evidence=evidence, + cost=cost, + hard_failed=(not judged.passed and judged.score <= 0.0), + ) + ) + + score = round(sum(case.score for case in case_results) / len(case_results), 6) if case_results else 0.0 + total_cost = round(sum(case.cost for case in case_results), 6) + return EvalResult( + prompt_id=prompt_id, + split=split, + score=score, + passed=all(case.passed for case in case_results), + cost=total_cost, + cases=case_results, + ) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py new file mode 100644 index 00000000..118001b2 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py @@ -0,0 +1,132 @@ +"""Deterministic fake judge for offline scoring.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from .schemas import EvalCase + + +@dataclass(frozen=True) +class JudgeOutcome: + score: float + passed: bool + error_code: str | None = None + evidence: str | None = None + trace: dict[str, Any] | None = None + + +class FakeJudge: + """Scores JSON, exact-answer, and rubric cases without calling an LLM.""" + + def score(self, case: EvalCase, output: str) -> JudgeOutcome: + expectation_type = case.expectation.get("type") + if expectation_type == "json": + return self._score_json(case, output) + if expectation_type == "exact": + return self._score_exact(case, output) + if expectation_type == "rubric": + return self._score_rubric(case, output) + return JudgeOutcome( + score=0.0, + passed=False, + error_code="unknown_expectation", + evidence=f"unsupported expectation type: {expectation_type!r}", + trace={"expectation_type": expectation_type}, + ) + + def _score_json(self, case: EvalCase, output: str) -> JudgeOutcome: + try: + parsed = json.loads(output) + except json.JSONDecodeError as exc: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_parse_failure", + evidence=f"json parser failed at char {exc.pos}: {exc.msg}", + trace={"expectation_type": "json", "valid_json": False}, + ) + if not isinstance(parsed, dict): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_value_mismatch", + evidence=f"expected JSON object, got {type(parsed).__name__}", + trace={"expectation_type": "json", "valid_json": True, "object": False}, + ) + required_keys = list(case.expectation.get("required_keys") or []) + for key in required_keys: + if key not in parsed: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="required_key_missing", + evidence=f"missing key {key!r}; got keys {sorted(parsed.keys())!r}", + trace={"expectation_type": "json", "valid_json": True, "missing_key": key}, + ) + expected_values = dict(case.expectation.get("expected_values") or {}) + for key, expected_value in expected_values.items(): + actual_value = parsed.get(key) + if actual_value != expected_value: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_value_mismatch", + evidence=f"{key!r}: expected {expected_value!r}, got {actual_value!r}", + trace={"expectation_type": "json", "valid_json": True, "mismatch_key": key}, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "json", "valid_json": True}) + + def _score_exact(self, case: EvalCase, output: str) -> JudgeOutcome: + expected = str(case.expectation.get("expected", "")) + if _normalize_exact(output) != _normalize_exact(expected): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="exact_answer_mismatch", + evidence=f"expected normalized {expected!r}, got {output!r}", + trace={"expectation_type": "exact", "expected": expected}, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "exact", "expected": expected}) + + def _score_rubric(self, case: EvalCase, output: str) -> JudgeOutcome: + lowered = output.lower() + forbidden = [str(item) for item in case.expectation.get("forbidden") or []] + for pattern in forbidden: + if pattern.lower() in lowered: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="forbidden_pattern", + evidence=f"forbidden pattern {pattern!r} was present", + trace={"expectation_type": "rubric", "forbidden_pattern": pattern}, + ) + + must_include = [str(item) for item in case.expectation.get("must_include") or []] + missing = [term for term in must_include if term.lower() not in lowered] + if missing: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="missing_rubric_terms", + evidence=f"missing terms: {missing!r}", + trace={"expectation_type": "rubric", "missing_terms": missing}, + ) + + max_chars = case.expectation.get("max_chars") + if max_chars is not None and len(output) > int(max_chars): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="max_chars_exceeded", + evidence=f"length {len(output)} exceeded max_chars {max_chars}", + trace={"expectation_type": "rubric", "length": len(output), "max_chars": int(max_chars)}, + ) + + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "rubric"}) + + +def _normalize_exact(value: str) -> str: + return " ".join(value.strip().lower().split()) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py new file mode 100644 index 00000000..6db9ef79 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py @@ -0,0 +1,99 @@ +"""Deterministic fake model used by the example pipeline.""" + +from __future__ import annotations + +import json +from typing import Any + +from .schemas import EvalCase + + +class FakeModel: + """Prompt-sensitive deterministic model. + + Markers injected by ``FakeOptimizer`` select behavior: + - no marker: baseline; adds prose around strict JSON/exact outputs. + - ALWAYS_OUTPUT_JSON: overfits to train formatting and forces JSON on + validation natural-language/exact cases. + - STRICT_WHEN_REQUESTED: applies JSON/exact constraints only when the case + explicitly asks for them. + """ + + COST_PER_CALL = 0.001 + + def __init__(self, seed: int = 91) -> None: + self.seed = seed + + def generate(self, prompt_id: str, prompt: str, case: EvalCase) -> tuple[str, dict[str, Any], float]: + mode = self._mode(prompt) + if mode == "overfit": + output = self._overfit_output(case) + elif mode == "safe": + output = self._safe_output(case) + else: + output = self._baseline_output(case) + trace = { + "seed": self.seed, + "prompt_id": prompt_id, + "prompt_mode": mode, + "case_id": case.case_id, + "expectation_type": case.expectation.get("type"), + } + return output, trace, self.COST_PER_CALL + + def _mode(self, prompt: str) -> str: + if "ALWAYS_OUTPUT_JSON" in prompt: + return "overfit" + if "STRICT_WHEN_REQUESTED" in prompt: + return "safe" + return "baseline" + + def _baseline_output(self, case: EvalCase) -> str: + expectation_type = case.expectation.get("type") + if expectation_type == "json": + return f"Here is the JSON you requested: {self._expected_json(case)}" + if expectation_type == "exact": + expected = str(case.expectation.get("expected", "")) + if case.case_id == "val_protected_yes_no": + return expected + return f"{expected} - confirmed." + if case.case_id == "train_rubric_retry_summary": + return "Latency improved because retries handle transient failures." + if case.case_id == "val_explain_cache": + return "A cache keeps recent data fast, but stale data can appear after updates." + return "I need more information." + + def _overfit_output(self, case: EvalCase) -> str: + expectation_type = case.expectation.get("type") + if case.split == "train": + return self._ideal_output(case) + if expectation_type == "json": + return self._expected_json(case) + if expectation_type == "exact": + return json.dumps({"answer": str(case.expectation.get("expected", ""))}, sort_keys=True) + return json.dumps({"answer": "Cache invalidation prevents stale data."}, sort_keys=True) + + def _safe_output(self, case: EvalCase) -> str: + user_asked = case.input.lower() + expectation_type = case.expectation.get("type") + if expectation_type == "json" or "json" in user_asked: + return self._expected_json(case) + if expectation_type == "exact" or "exactly" in user_asked: + return str(case.expectation.get("expected", "")) + return self._ideal_output(case) + + def _ideal_output(self, case: EvalCase) -> str: + expectation_type = case.expectation.get("type") + if expectation_type == "json": + return self._expected_json(case) + if expectation_type == "exact": + return str(case.expectation.get("expected", "")) + if case.case_id == "train_rubric_retry_summary": + return "Latency improved because retries handle transient failures." + if case.case_id == "val_explain_cache": + return "A cache keeps recent data fast, but stale data can appear after updates." + return "Meets the rubric." + + def _expected_json(self, case: EvalCase) -> str: + values = dict(case.expectation.get("expected_values") or {}) + return json.dumps(values, sort_keys=True) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/gate.py b/examples/optimization/eval_optimize_loop/eval_loop/gate.py new file mode 100644 index 00000000..47ca61ae --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/gate.py @@ -0,0 +1,106 @@ +"""Configurable acceptance gate for candidate prompts.""" + +from __future__ import annotations + +from typing import Any + +from .schemas import CaseDelta +from .schemas import EvalResult +from .schemas import GateDecision + + +DEFAULT_GATE_CONFIG = { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, +} + + +class AcceptanceGate: + """Apply deterministic safety and quality constraints to candidates.""" + + def __init__(self, config: dict[str, Any]) -> None: + merged = dict(DEFAULT_GATE_CONFIG) + merged.update(config or {}) + self.config = merged + + def decide( + self, + *, + candidate_id: str, + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_train: EvalResult, + candidate_validation: EvalResult, + deltas: list[CaseDelta], + ) -> GateDecision: + train_delta = round(candidate_train.score - baseline_train.score, 6) + val_delta = round(candidate_validation.score - baseline_validation.score, 6) + candidate_cost = round(candidate_train.cost + candidate_validation.cost, 6) + reasons: list[str] = [] + + if train_delta > 0 and val_delta < 0: + reasons.append( + "reject: train score improved but validation score regressed " + f"({train_delta:+.3f} train, {val_delta:+.3f} validation)" + ) + + min_val_improvement = float(self.config["min_val_score_improvement"]) + if val_delta < min_val_improvement: + reasons.append( + "reject: validation improvement " + f"{val_delta:+.3f} is below required {min_val_improvement:+.3f}" + ) + + baseline_validation_by_id = baseline_validation.by_case_id() + candidate_validation_by_id = candidate_validation.by_case_id() + new_hard_failures = [ + case_id + for case_id, candidate_case in sorted(candidate_validation_by_id.items()) + if candidate_case.hard_failed and baseline_validation_by_id.get(case_id) + and baseline_validation_by_id[case_id].passed + ] + if new_hard_failures and not bool(self.config["allow_new_hard_fail"]): + reasons.append(f"reject: new hard failures appeared: {new_hard_failures}") + + protected_ids = set(str(item) for item in self.config["protected_case_ids"]) + protected_regressions = [ + delta.case_id + for delta in deltas + if delta.split == "validation" and delta.case_id in protected_ids and delta.delta < 0 + ] + if protected_regressions: + reasons.append(f"reject: protected cases regressed: {protected_regressions}") + + max_drop = float(self.config["max_score_drop_per_case"]) + excessive_drops = [ + delta.case_id + for delta in deltas + if delta.split == "validation" and delta.delta < -max_drop + ] + if excessive_drops: + reasons.append(f"reject: per-case validation score drops exceed {max_drop:.3f}: {excessive_drops}") + + max_total_cost = float(self.config["max_total_cost"]) + if candidate_cost > max_total_cost: + reasons.append(f"reject: candidate cost {candidate_cost:.3f} exceeds budget {max_total_cost:.3f}") + + accepted = not any(reason.startswith("reject:") for reason in reasons) + if accepted: + reasons.append( + "accept: validation score improved " + f"{val_delta:+.3f} with no protected regression or new hard failure" + ) + + return GateDecision( + candidate_id=candidate_id, + accepted=accepted, + reasons=reasons, + train_score_delta=train_delta, + validation_score_delta=val_delta, + new_hard_failures=new_hard_failures, + protected_regressions=protected_regressions, + cost=candidate_cost, + ) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/loader.py b/examples/optimization/eval_optimize_loop/eval_loop/loader.py new file mode 100644 index 00000000..61a2c780 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/loader.py @@ -0,0 +1,45 @@ +"""Input loading helpers for the deterministic optimization example.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .schemas import EvalCase + + +def read_json(path: str | Path) -> dict[str, Any]: + resolved = Path(path) + with resolved.open("r", encoding="utf-8") as file: + payload = json.load(file) + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object in {resolved}") + return payload + + +def load_eval_cases(path: str | Path, split: str | None = None) -> list[EvalCase]: + payload = read_json(path) + cases = payload.get("cases") + if not isinstance(cases, list): + raise ValueError(f"evalset {path} must contain a cases list") + effective_split = split or payload.get("split") or Path(path).name.split(".", 1)[0] + return [EvalCase.from_dict(case, str(effective_split)) for case in cases] + + +def load_optimizer_config(path: str | Path) -> dict[str, Any]: + payload = read_json(path) + if "gate" not in payload: + raise ValueError(f"optimizer config {path} must contain a gate object") + return payload + + +def load_prompt(path: str | Path) -> str: + return Path(path).read_text(encoding="utf-8") + + +def stable_config_hash(config: dict[str, Any]) -> str: + import hashlib + + canonical = json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() diff --git a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py new file mode 100644 index 00000000..ecb56391 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py @@ -0,0 +1,52 @@ +"""Fake optimizer that proposes deterministic prompt candidates.""" + +from __future__ import annotations + +from .schemas import CandidatePrompt + + +class FakeOptimizer: + """Produce the two candidates required by the example issue.""" + + def propose(self, baseline_prompt: str) -> list[CandidatePrompt]: + return [ + CandidatePrompt( + candidate_id="candidate_001_overfit", + prompt=( + baseline_prompt.rstrip() + + "\n\n" + + "# Optimizer patch\n" + + "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n" + + "Always force every final answer into JSON, even when the user asks for prose.\n" + ), + rationale=( + "The train failures are strict JSON/exact formatting failures, so this candidate " + "over-corrects by forcing JSON globally." + ), + prompt_diff=( + "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n" + "+ Always force every final answer into JSON, even when the user asks for prose." + ), + ), + CandidatePrompt( + candidate_id="candidate_002_safe", + prompt=( + baseline_prompt.rstrip() + + "\n\n" + + "# Optimizer patch\n" + + "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n" + + "Use strict JSON only when the user explicitly asks for JSON.\n" + + "Use exact answers only when the user explicitly asks for an exact answer.\n" + + "Otherwise answer naturally and honor rubric constraints.\n" + ), + rationale=( + "This candidate fixes observed strict-format failures without changing " + "natural-language behavior on validation cases." + ), + prompt_diff=( + "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n" + "+ Use strict JSON only when explicitly requested.\n" + "+ Preserve natural-language answers unless a strict format is requested." + ), + ), + ] diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py new file mode 100644 index 00000000..ea1d0e43 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -0,0 +1,198 @@ +"""Report construction and rendering.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .attribution import summarize_failures +from .schemas import CandidatePrompt +from .schemas import CaseDelta +from .schemas import EvalResult +from .schemas import GateDecision +from .schemas import OptimizationReport +from .schemas import to_jsonable + + +REPRODUCIBILITY_COMMAND = ( + "python examples/optimization/eval_optimize_loop/run_pipeline.py " + "--train examples/optimization/eval_optimize_loop/data/train.evalset.json " + "--val examples/optimization/eval_optimize_loop/data/val.evalset.json " + "--optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json " + "--prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt " + "--output-dir /tmp/eval-optimize-loop " + "--fake-model --fake-judge --trace" +) + + +def compute_case_deltas( + *, + candidate_id: str, + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_train: EvalResult, + candidate_validation: EvalResult, +) -> list[CaseDelta]: + deltas: list[CaseDelta] = [] + for baseline, candidate in ((baseline_train, candidate_train), (baseline_validation, candidate_validation)): + candidate_by_id = candidate.by_case_id() + for baseline_case in baseline.cases: + candidate_case = candidate_by_id[baseline_case.case_id] + delta = round(candidate_case.score - baseline_case.score, 6) + deltas.append( + CaseDelta( + candidate_id=candidate_id, + case_id=baseline_case.case_id, + split=baseline_case.split, + baseline_score=baseline_case.score, + candidate_score=candidate_case.score, + delta=delta, + baseline_passed=baseline_case.passed, + candidate_passed=candidate_case.passed, + regression=delta < 0, + ) + ) + return deltas + + +def build_report( + *, + run: dict[str, Any], + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_records: list[dict[str, Any]], + per_case_deltas: list[CaseDelta], + gate_decisions: list[GateDecision], + selected_candidate: str | None, + audit: dict[str, Any], +) -> OptimizationReport: + all_results: list[EvalResult] = [baseline_train, baseline_validation] + for record in candidate_records: + all_results.append(record["train_result"]) + all_results.append(record["validation_result"]) + return OptimizationReport( + schema_version="eval_optimize_loop.v1", + run=run, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidates=candidate_records, + per_case_deltas=per_case_deltas, + failure_attribution_summary=summarize_failures(all_results), + gate_decisions=gate_decisions, + selected_candidate=selected_candidate, + audit=audit, + ) + + +def write_reports(report: OptimizationReport, output_dir: str | Path) -> tuple[Path, Path]: + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + json_path = output_path / "optimization_report.json" + md_path = output_path / "optimization_report.md" + json_path.write_text(report_to_json(report), encoding="utf-8") + md_path.write_text(render_markdown(report), encoding="utf-8") + return json_path, md_path + + +def report_to_json(report: OptimizationReport) -> str: + return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True) + "\n" + + +def render_markdown(report: OptimizationReport) -> str: + decision_by_id = {decision.candidate_id: decision for decision in report.gate_decisions} + lines = [ + "# Evaluation + Optimization Report", + "", + "## Final Decision", + "", + ] + if report.selected_candidate: + lines.append(f"Selected candidate: `{report.selected_candidate}`.") + else: + lines.append("No candidate was accepted.") + + lines.extend([ + "", + "## Gate Reasons", + "", + ]) + for decision in report.gate_decisions: + verdict = "accepted" if decision.accepted else "rejected" + lines.append(f"### {decision.candidate_id} ({verdict})") + for reason in decision.reasons: + lines.append(f"- {reason}") + lines.append("") + + lines.extend([ + "## Baseline vs Candidate Scores", + "", + "| prompt | train score | validation score | gate |", + "| --- | ---: | ---: | --- |", + f"| baseline | {report.baseline_train.score:.3f} | {report.baseline_validation.score:.3f} | n/a |", + ]) + for record in report.candidates: + candidate: CandidatePrompt = record["candidate"] + gate = decision_by_id[candidate.candidate_id] + verdict = "accept" if gate.accepted else "reject" + lines.append( + f"| {candidate.candidate_id} | {record['train_result'].score:.3f} | " + f"{record['validation_result'].score:.3f} | {verdict} |" + ) + + lines.extend([ + "", + "## Per-Case Delta", + "", + "| candidate | split | case | baseline | candidate | delta | passed -> passed |", + "| --- | --- | --- | ---: | ---: | ---: | --- |", + ]) + for delta in report.per_case_deltas: + lines.append( + f"| {delta.candidate_id} | {delta.split} | {delta.case_id} | " + f"{delta.baseline_score:.3f} | {delta.candidate_score:.3f} | " + f"{delta.delta:+.3f} | {delta.baseline_passed} -> {delta.candidate_passed} |" + ) + + summary = report.failure_attribution_summary + lines.extend([ + "", + "## Failure Attribution Summary", + "", + f"Total failed case evaluations: {summary['total_failed_cases']}", + "", + "| category | count |", + "| --- | ---: |", + ]) + by_category = summary.get("by_category", {}) + if by_category: + for category, count in by_category.items(): + lines.append(f"| {category} | {count} |") + else: + lines.append("| none | 0 |") + + lines.extend([ + "", + "## Prompt Diff", + "", + ]) + for record in report.candidates: + candidate = record["candidate"] + lines.extend([ + f"### {candidate.candidate_id}", + "", + "```diff", + candidate.prompt_diff, + "```", + "", + ]) + + lines.extend([ + "## Reproducibility", + "", + "```bash", + REPRODUCIBILITY_COMMAND, + "```", + "", + ]) + return "\n".join(lines) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py new file mode 100644 index 00000000..31d43cba --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -0,0 +1,139 @@ +"""Dataclass schemas used by the example optimization loop.""" + +from __future__ import annotations + +from dataclasses import asdict +from dataclasses import dataclass +from dataclasses import field +from dataclasses import is_dataclass +from typing import Any + + +@dataclass(frozen=True) +class EvalCase: + """One deterministic evaluation case.""" + + case_id: str + split: str + input: str + expectation: dict[str, Any] + tags: list[str] = field(default_factory=list) + protected: bool = False + + @classmethod + def from_dict(cls, payload: dict[str, Any], split: str) -> "EvalCase": + case_id = payload.get("case_id") or payload.get("id") + if not case_id: + raise ValueError(f"eval case is missing id/case_id: {payload!r}") + expectation = payload.get("expectation") + if not isinstance(expectation, dict): + raise ValueError(f"eval case {case_id!r} is missing expectation object") + return cls( + case_id=str(case_id), + split=str(payload.get("split") or split), + input=str(payload.get("input") or payload.get("user_input") or ""), + expectation=dict(expectation), + tags=list(payload.get("tags") or []), + protected=bool(payload.get("protected", False)), + ) + + +@dataclass(frozen=True) +class CaseResult: + """Evaluation result for one case under one prompt.""" + + case_id: str + split: str + score: float + passed: bool + output: str + trace: dict[str, Any] = field(default_factory=dict) + failure_category: str | None = None + failure_reason: str | None = None + evidence: str | None = None + cost: float = 0.0 + hard_failed: bool = False + + +@dataclass(frozen=True) +class EvalResult: + """Aggregate result for one prompt on one split.""" + + prompt_id: str + split: str + score: float + passed: bool + cost: float + cases: list[CaseResult] + + def by_case_id(self) -> dict[str, CaseResult]: + return {case.case_id: case for case in self.cases} + + +@dataclass(frozen=True) +class CandidatePrompt: + """A proposed prompt candidate.""" + + candidate_id: str + prompt: str + rationale: str + prompt_diff: str + + +@dataclass(frozen=True) +class CaseDelta: + """Per-case score delta from baseline to candidate.""" + + candidate_id: str + case_id: str + split: str + baseline_score: float + candidate_score: float + delta: float + baseline_passed: bool + candidate_passed: bool + regression: bool + + +@dataclass(frozen=True) +class GateDecision: + """Configurable acceptance gate result for one candidate.""" + + candidate_id: str + accepted: bool + reasons: list[str] + train_score_delta: float + validation_score_delta: float + new_hard_failures: list[str] + protected_regressions: list[str] + cost: float + + +@dataclass(frozen=True) +class OptimizationReport: + """Complete persisted audit report for the loop.""" + + schema_version: str + run: dict[str, Any] + baseline_train: EvalResult + baseline_validation: EvalResult + candidates: list[dict[str, Any]] + per_case_deltas: list[CaseDelta] + failure_attribution_summary: dict[str, Any] + gate_decisions: list[GateDecision] + selected_candidate: str | None + audit: dict[str, Any] + + +def to_jsonable(value: Any) -> Any: + """Convert dataclasses and nested containers into JSON-serializable data.""" + + if is_dataclass(value): + return {key: to_jsonable(item) for key, item in asdict(value).items()} + if isinstance(value, dict): + return {str(key): to_jsonable(item) for key, item in value.items()} + if isinstance(value, list): + return [to_jsonable(item) for item in value] + if isinstance(value, tuple): + return [to_jsonable(item) for item in value] + return value diff --git a/examples/optimization/eval_optimize_loop/eval_loop/trace.py b/examples/optimization/eval_optimize_loop/eval_loop/trace.py new file mode 100644 index 00000000..917416d5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/trace.py @@ -0,0 +1,18 @@ +"""Trace helpers for fake-mode runs.""" + +from __future__ import annotations + +from typing import Any + + +def make_trace(enabled: bool, *, prompt_id: str, case_id: str, model_trace: dict[str, Any], + judge_trace: dict[str, Any]) -> dict[str, Any]: + if not enabled: + return {} + return { + "trace_mode": "fake", + "prompt_id": prompt_id, + "case_id": case_id, + "model": model_trace, + "judge": judge_trace, + } diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json new file mode 100644 index 00000000..caea392d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json @@ -0,0 +1,845 @@ +{ + "audit": { + "candidate_prompts": { + "candidate_001_overfit": { + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", + "prompt_diff": "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+ Always force every final answer into JSON, even when the user asks for prose.", + "rationale": "The train failures are strict JSON/exact formatting failures, so this candidate over-corrects by forcing JSON globally." + }, + "candidate_002_safe": { + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\nUse strict JSON only when the user explicitly asks for JSON.\nUse exact answers only when the user explicitly asks for an exact answer.\nOtherwise answer naturally and honor rubric constraints.\n", + "prompt_diff": "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+ Use strict JSON only when explicitly requested.\n+ Preserve natural-language answers unless a strict format is requested.", + "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." + } + }, + "config_hash": "1fafb751c9d2b6c63b0daff25844ce64f7f49da32717855ce9b3c50206a6ebe0", + "cost": { + "baseline": 0.006, + "candidates": { + "candidate_001_overfit": 0.006, + "candidate_002_safe": 0.006 + }, + "total": 0.018 + }, + "duration_seconds": 0.0, + "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", + "seed": 91 + }, + "baseline_train": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "output": "Here is the JSON you requested: {\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "case_id": "train_json_refund", + "judge": { + "expectation_type": "json", + "valid_json": false + }, + "model": { + "case_id": "train_json_refund", + "expectation_type": "json", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": "expected normalized 'READY', got 'READY - confirmed.'", + "failure_category": "final_answer_mismatch", + "failure_reason": "normalized exact answer mismatch", + "hard_failed": true, + "output": "READY - confirmed.", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "case_id": "train_exact_order_status", + "judge": { + "expectation_type": "exact", + "expected": "READY" + }, + "model": { + "case_id": "train_exact_order_status", + "expectation_type": "exact", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "Latency improved because retries handle transient failures.", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_rubric_retry_summary", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "train_rubric_retry_summary", + "expectation_type": "rubric", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.333333, + "split": "train" + }, + "baseline_validation": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "output": "Here is the JSON you requested: {\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "case_id": "val_json_invoice", + "judge": { + "expectation_type": "json", + "valid_json": false + }, + "model": { + "case_id": "val_json_invoice", + "expectation_type": "json", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "A cache keeps recent data fast, but stale data can appear after updates.", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_explain_cache", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "val_explain_cache", + "expectation_type": "rubric", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_protected_yes_no", + "judge": { + "expectation_type": "exact", + "expected": "YES" + }, + "model": { + "case_id": "val_protected_yes_no", + "expectation_type": "exact", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.666667, + "split": "validation" + }, + "candidates": [ + { + "candidate": { + "candidate_id": "candidate_001_overfit", + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", + "prompt_diff": "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+ Always force every final answer into JSON, even when the user asks for prose.", + "rationale": "The train failures are strict JSON/exact formatting failures, so this candidate over-corrects by forcing JSON globally." + }, + "train_result": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "{\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_json_refund", + "judge": { + "expectation_type": "json", + "valid_json": true + }, + "model": { + "case_id": "train_json_refund", + "expectation_type": "json", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "READY", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_exact_order_status", + "judge": { + "expectation_type": "exact", + "expected": "READY" + }, + "model": { + "case_id": "train_exact_order_status", + "expectation_type": "exact", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "Latency improved because retries handle transient failures.", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_rubric_retry_summary", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "train_rubric_retry_summary", + "expectation_type": "rubric", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_001_overfit", + "score": 1.0, + "split": "train" + }, + "validation_result": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "{\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_json_invoice", + "judge": { + "expectation_type": "json", + "valid_json": true + }, + "model": { + "case_id": "val_json_invoice", + "expectation_type": "json", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": "forbidden pattern '{' was present", + "failure_category": "format_violation", + "failure_reason": "output contains a forbidden pattern", + "hard_failed": true, + "output": "{\"answer\": \"Cache invalidation prevents stale data.\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "case_id": "val_explain_cache", + "judge": { + "expectation_type": "rubric", + "forbidden_pattern": "{" + }, + "model": { + "case_id": "val_explain_cache", + "expectation_type": "rubric", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": "expected normalized 'YES', got '{\"answer\": \"YES\"}'", + "failure_category": "final_answer_mismatch", + "failure_reason": "normalized exact answer mismatch", + "hard_failed": true, + "output": "{\"answer\": \"YES\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "case_id": "val_protected_yes_no", + "judge": { + "expectation_type": "exact", + "expected": "YES" + }, + "model": { + "case_id": "val_protected_yes_no", + "expectation_type": "exact", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "candidate_001_overfit", + "score": 0.333333, + "split": "validation" + } + }, + { + "candidate": { + "candidate_id": "candidate_002_safe", + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\nUse strict JSON only when the user explicitly asks for JSON.\nUse exact answers only when the user explicitly asks for an exact answer.\nOtherwise answer naturally and honor rubric constraints.\n", + "prompt_diff": "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+ Use strict JSON only when explicitly requested.\n+ Preserve natural-language answers unless a strict format is requested.", + "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." + }, + "train_result": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "{\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_json_refund", + "judge": { + "expectation_type": "json", + "valid_json": true + }, + "model": { + "case_id": "train_json_refund", + "expectation_type": "json", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "READY", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_exact_order_status", + "judge": { + "expectation_type": "exact", + "expected": "READY" + }, + "model": { + "case_id": "train_exact_order_status", + "expectation_type": "exact", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "Latency improved because retries handle transient failures.", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_rubric_retry_summary", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "train_rubric_retry_summary", + "expectation_type": "rubric", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_002_safe", + "score": 1.0, + "split": "train" + }, + "validation_result": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "{\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_json_invoice", + "judge": { + "expectation_type": "json", + "valid_json": true + }, + "model": { + "case_id": "val_json_invoice", + "expectation_type": "json", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "A cache keeps recent data fast, but stale data can appear after updates.", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_explain_cache", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "val_explain_cache", + "expectation_type": "rubric", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_protected_yes_no", + "judge": { + "expectation_type": "exact", + "expected": "YES" + }, + "model": { + "case_id": "val_protected_yes_no", + "expectation_type": "exact", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_002_safe", + "score": 1.0, + "split": "validation" + } + } + ], + "failure_attribution_summary": { + "by_category": { + "final_answer_mismatch": 2, + "format_violation": 3 + }, + "by_prompt_split": { + "baseline:train": { + "final_answer_mismatch": 1, + "format_violation": 1 + }, + "baseline:validation": { + "format_violation": 1 + }, + "candidate_001_overfit:train": {}, + "candidate_001_overfit:validation": { + "final_answer_mismatch": 1, + "format_violation": 1 + }, + "candidate_002_safe:train": {}, + "candidate_002_safe:validation": {} + }, + "examples": [ + { + "case_id": "train_json_refund", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "train" + }, + { + "case_id": "train_exact_order_status", + "evidence": "expected normalized 'READY', got 'READY - confirmed.'", + "failure_category": "final_answer_mismatch", + "failure_reason": "normalized exact answer mismatch", + "prompt_id": "baseline", + "split": "train" + }, + { + "case_id": "val_json_invoice", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "validation" + }, + { + "case_id": "val_explain_cache", + "evidence": "forbidden pattern '{' was present", + "failure_category": "format_violation", + "failure_reason": "output contains a forbidden pattern", + "prompt_id": "candidate_001_overfit", + "split": "validation" + }, + { + "case_id": "val_protected_yes_no", + "evidence": "expected normalized 'YES', got '{\"answer\": \"YES\"}'", + "failure_category": "final_answer_mismatch", + "failure_reason": "normalized exact answer mismatch", + "prompt_id": "candidate_001_overfit", + "split": "validation" + } + ], + "total_failed_cases": 5 + }, + "gate_decisions": [ + { + "accepted": false, + "candidate_id": "candidate_001_overfit", + "cost": 0.006, + "new_hard_failures": [ + "val_explain_cache", + "val_protected_yes_no" + ], + "protected_regressions": [ + "val_protected_yes_no" + ], + "reasons": [ + "reject: train score improved but validation score regressed (+0.667 train, -0.333 validation)", + "reject: validation improvement -0.333 is below required +0.010", + "reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no']", + "reject: protected cases regressed: ['val_protected_yes_no']", + "reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no']" + ], + "train_score_delta": 0.666667, + "validation_score_delta": -0.333334 + }, + { + "accepted": true, + "candidate_id": "candidate_002_safe", + "cost": 0.006, + "new_hard_failures": [], + "protected_regressions": [], + "reasons": [ + "accept: validation score improved +0.333 with no protected regression or new hard failure" + ], + "train_score_delta": 0.666667, + "validation_score_delta": 0.333333 + } + ], + "per_case_deltas": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_explain_cache", + "delta": -1.0, + "regression": true, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_protected_yes_no", + "delta": -1.0, + "regression": true, + "split": "validation" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_explain_cache", + "delta": 0.0, + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_protected_yes_no", + "delta": 0.0, + "regression": false, + "split": "validation" + } + ], + "run": { + "fake_judge": true, + "fake_model": true, + "mode": "fake", + "prompt_source": "prompts/baseline_system_prompt.txt", + "run_id": "eval_optimize_loop_seed_91", + "trace_enabled": true, + "train_cases": 3, + "validation_cases": 3 + }, + "schema_version": "eval_optimize_loop.v1", + "selected_candidate": "candidate_002_safe" +} diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md new file mode 100644 index 00000000..7c5fc8cf --- /dev/null +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md @@ -0,0 +1,74 @@ +# Evaluation + Optimization Report + +## Final Decision + +Selected candidate: `candidate_002_safe`. + +## Gate Reasons + +### candidate_001_overfit (rejected) +- reject: train score improved but validation score regressed (+0.667 train, -0.333 validation) +- reject: validation improvement -0.333 is below required +0.010 +- reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no'] +- reject: protected cases regressed: ['val_protected_yes_no'] +- reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no'] + +### candidate_002_safe (accepted) +- accept: validation score improved +0.333 with no protected regression or new hard failure + +## Baseline vs Candidate Scores + +| prompt | train score | validation score | gate | +| --- | ---: | ---: | --- | +| baseline | 0.333 | 0.667 | n/a | +| candidate_001_overfit | 1.000 | 0.333 | reject | +| candidate_002_safe | 1.000 | 1.000 | accept | + +## Per-Case Delta + +| candidate | split | case | baseline | candidate | delta | passed -> passed | +| --- | --- | --- | ---: | ---: | ---: | --- | +| candidate_001_overfit | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | +| candidate_001_overfit | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | +| candidate_001_overfit | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | +| candidate_001_overfit | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | +| candidate_001_overfit | validation | val_explain_cache | 1.000 | 0.000 | -1.000 | True -> False | +| candidate_001_overfit | validation | val_protected_yes_no | 1.000 | 0.000 | -1.000 | True -> False | +| candidate_002_safe | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | +| candidate_002_safe | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | +| candidate_002_safe | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | +| candidate_002_safe | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | +| candidate_002_safe | validation | val_explain_cache | 1.000 | 1.000 | +0.000 | True -> True | +| candidate_002_safe | validation | val_protected_yes_no | 1.000 | 1.000 | +0.000 | True -> True | + +## Failure Attribution Summary + +Total failed case evaluations: 5 + +| category | count | +| --- | ---: | +| final_answer_mismatch | 2 | +| format_violation | 3 | + +## Prompt Diff + +### candidate_001_overfit + +```diff ++ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON ++ Always force every final answer into JSON, even when the user asks for prose. +``` + +### candidate_002_safe + +```diff ++ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED ++ Use strict JSON only when explicitly requested. ++ Preserve natural-language answers unless a strict format is requested. +``` + +## Reproducibility + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace +``` diff --git a/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt b/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt new file mode 100644 index 00000000..7b3ed8aa --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt @@ -0,0 +1,4 @@ +You are a helpful support assistant. + +Answer clearly and include a short explanation when it may help the user. +If the user asks for structured data, provide the information they need. diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..23f7ec20 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,253 @@ +"""Run the deterministic Evaluation + Optimization closed-loop example.""" + +from __future__ import annotations + +import argparse +import sys +import tempfile +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +from eval_loop.evaluator import ExampleEvaluator +from eval_loop.fake_judge import FakeJudge +from eval_loop.fake_model import FakeModel +from eval_loop.gate import AcceptanceGate +from eval_loop.loader import load_eval_cases +from eval_loop.loader import load_optimizer_config +from eval_loop.loader import load_prompt +from eval_loop.loader import stable_config_hash +from eval_loop.optimizer import FakeOptimizer +from eval_loop.report import REPRODUCIBILITY_COMMAND +from eval_loop.report import build_report +from eval_loop.report import compute_case_deltas +from eval_loop.report import write_reports +from eval_loop.schemas import CandidatePrompt +from eval_loop.schemas import OptimizationReport + + +DEFAULT_TRAIN = HERE / "data" / "train.evalset.json" +DEFAULT_VAL = HERE / "data" / "val.evalset.json" +DEFAULT_OPTIMIZER_CONFIG = HERE / "data" / "optimizer.json" +DEFAULT_PROMPT = HERE / "prompts" / "baseline_system_prompt.txt" +DEFAULT_OUTPUT_DIR = Path(tempfile.gettempdir()) / "eval-optimize-loop" + + +def run_pipeline( + *, + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, +) -> OptimizationReport: + """Run baseline eval, fake optimization, validation gate, and reports.""" + + if not fake_model or not fake_judge: + raise ValueError( + "This example-local implementation only supports offline fake mode. " + "Pass --fake-model --fake-judge, or replace ExampleEvaluator with a real adapter." + ) + + train_cases = load_eval_cases(train_path, split="train") + validation_cases = load_eval_cases(val_path, split="validation") + if len(train_cases) < 3 or len(validation_cases) < 3: + raise ValueError("example requires at least 3 train and 3 validation eval cases") + + optimizer_config = load_optimizer_config(optimizer_config_path) + seed = int(optimizer_config.get("seed", 91)) + baseline_prompt = load_prompt(prompt_path) + evaluator = ExampleEvaluator(FakeModel(seed=seed), FakeJudge(), trace_enabled=trace) + + baseline = CandidatePrompt( + candidate_id="baseline", + prompt=baseline_prompt, + rationale="Prompt source file before optimization.", + prompt_diff="", + ) + baseline_train = evaluator.evaluate( + prompt_id=baseline.candidate_id, + prompt=baseline.prompt, + cases=train_cases, + split="train", + ) + baseline_validation = evaluator.evaluate( + prompt_id=baseline.candidate_id, + prompt=baseline.prompt, + cases=validation_cases, + split="validation", + ) + + optimizer = FakeOptimizer() + candidates = optimizer.propose(baseline_prompt) + gate = AcceptanceGate(optimizer_config["gate"]) + + candidate_records: list[dict[str, Any]] = [] + all_deltas = [] + gate_decisions = [] + for candidate in candidates: + train_result = evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompt=candidate.prompt, + cases=train_cases, + split="train", + ) + validation_result = evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompt=candidate.prompt, + cases=validation_cases, + split="validation", + ) + deltas = compute_case_deltas( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + ) + decision = gate.decide( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + deltas=deltas, + ) + candidate_records.append({ + "candidate": candidate, + "train_result": train_result, + "validation_result": validation_result, + }) + all_deltas.extend(deltas) + gate_decisions.append(decision) + + selected_candidate = _select_candidate(candidate_records, gate_decisions) + audit = _build_audit( + seed=seed, + config_hash=stable_config_hash(optimizer_config), + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + candidates=candidates, + ) + run = { + "run_id": f"eval_optimize_loop_seed_{seed}", + "mode": "fake", + "fake_model": fake_model, + "fake_judge": fake_judge, + "trace_enabled": trace, + "train_cases": len(train_cases), + "validation_cases": len(validation_cases), + "prompt_source": "prompts/baseline_system_prompt.txt", + } + report = build_report( + run=run, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + per_case_deltas=all_deltas, + gate_decisions=gate_decisions, + selected_candidate=selected_candidate, + audit=audit, + ) + write_reports(report, output_dir) + return report + + +def _select_candidate(candidate_records: list[dict[str, Any]], gate_decisions: list) -> str | None: + decisions_by_id = {decision.candidate_id: decision for decision in gate_decisions} + accepted = [] + for index, record in enumerate(candidate_records): + candidate = record["candidate"] + decision = decisions_by_id[candidate.candidate_id] + if decision.accepted: + accepted.append((index, record)) + if not accepted: + return None + index, record = max( + accepted, + key=lambda item: ( + item[1]["validation_result"].score, + item[1]["train_result"].score, + -item[0], + ), + ) + return record["candidate"].candidate_id + + +def _build_audit( + *, + seed: int, + config_hash: str, + baseline_train, + baseline_validation, + candidate_records: list[dict[str, Any]], + candidates: list[CandidatePrompt], +) -> dict[str, Any]: + baseline_cost = round(baseline_train.cost + baseline_validation.cost, 6) + candidate_costs = { + record["candidate"].candidate_id: round(record["train_result"].cost + record["validation_result"].cost, 6) + for record in candidate_records + } + total_cost = round(baseline_cost + sum(candidate_costs.values()), 6) + return { + "seed": seed, + "duration_seconds": 0.0, + "config_hash": config_hash, + "cost": { + "baseline": baseline_cost, + "candidates": candidate_costs, + "total": total_cost, + }, + "candidate_prompts": { + candidate.candidate_id: { + "rationale": candidate.rationale, + "prompt": candidate.prompt, + "prompt_diff": candidate.prompt_diff, + } + for candidate in candidates + }, + "reproducibility_command": REPRODUCIBILITY_COMMAND, + } + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--train", default=str(DEFAULT_TRAIN), help="Path to train.evalset.json") + parser.add_argument("--val", default=str(DEFAULT_VAL), help="Path to val.evalset.json") + parser.add_argument("--optimizer-config", default=str(DEFAULT_OPTIMIZER_CONFIG), help="Path to optimizer.json") + parser.add_argument("--prompt", default=str(DEFAULT_PROMPT), help="Path to baseline system prompt") + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR), help="Directory for runtime reports") + parser.add_argument("--fake-model", action="store_true", help="Use deterministic fake model") + parser.add_argument("--fake-judge", action="store_true", help="Use deterministic fake judge") + parser.add_argument("--trace", action="store_true", help="Persist fake model/judge trace details per case") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> OptimizationReport: + args = parse_args(argv) + report = run_pipeline( + train_path=args.train, + val_path=args.val, + optimizer_config_path=args.optimizer_config, + prompt_path=args.prompt, + output_dir=args.output_dir, + fake_model=args.fake_model, + fake_judge=args.fake_judge, + trace=args.trace, + ) + output_dir = Path(args.output_dir) + print(f"Wrote {output_dir / 'optimization_report.json'}") + print(f"Wrote {output_dir / 'optimization_report.md'}") + print(f"Selected candidate: {report.selected_candidate}") + return report + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py new file mode 100644 index 00000000..22356be8 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from examples.optimization.eval_optimize_loop.eval_loop.attribution import attribute_failure +from examples.optimization.eval_optimize_loop.eval_loop.evaluator import ExampleEvaluator +from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase + + +def test_failure_attribution_labels_format_violation_and_exact_mismatch(): + assert attribute_failure("json_parse_failure", "bad json")[0] == "format_violation" + assert attribute_failure("exact_answer_mismatch", "wrong answer")[0] == "final_answer_mismatch" + + +def test_evaluator_attaches_failure_details_to_failed_cases(): + cases = [ + EvalCase( + case_id="json_case", + split="train", + input="Return JSON.", + expectation={ + "type": "json", + "required_keys": ["answer"], + "expected_values": {"answer": "ok"}, + }, + ), + EvalCase( + case_id="exact_case", + split="train", + input="Answer exactly OK.", + expectation={"type": "exact", "expected": "OK"}, + ), + ] + result = ExampleEvaluator(FakeModel(seed=91), FakeJudge()).evaluate( + prompt_id="baseline", + prompt="baseline prompt", + cases=cases, + split="train", + ) + + by_id = result.by_case_id() + assert by_id["json_case"].failure_category == "format_violation" + assert by_id["json_case"].failure_reason + assert by_id["json_case"].evidence + assert by_id["exact_case"].failure_category == "final_answer_mismatch" + assert by_id["exact_case"].failure_reason + assert by_id["exact_case"].evidence diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py new file mode 100644 index 00000000..041d4ec1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from examples.optimization.eval_optimize_loop.eval_loop.gate import AcceptanceGate +from examples.optimization.eval_optimize_loop.eval_loop.report import compute_case_deltas +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult + + +def test_gate_rejects_train_improvement_with_validation_regression(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0), ("train_b", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0), ("train_b", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 1.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0), ("val_b", 0.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({"protected_case_ids": []}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert not decision.accepted + assert any("train score improved but validation score regressed" in reason for reason in decision.reasons) + + +def test_gate_rejects_protected_case_regression(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("protected", 1.0), ("val_a", 0.0), ("val_b", 0.0)]) + candidate_val = _eval("candidate", "validation", [("protected", 0.0), ("val_a", 1.0), ("val_b", 1.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({"protected_case_ids": ["protected"]}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert not decision.accepted + assert decision.protected_regressions == ["protected"] + assert any("protected cases regressed" in reason for reason in decision.reasons) + + +def test_gate_accepts_safe_candidate(): + baseline_train = _eval("baseline", "train", [("train_json", 0.0), ("train_exact", 0.0), ("train_ok", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_json", 1.0), ("train_exact", 1.0), ("train_ok", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_json", 0.0), ("val_text", 1.0), ("protected", 1.0)]) + candidate_val = _eval("candidate", "validation", [("val_json", 1.0), ("val_text", 1.0), ("protected", 1.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({"protected_case_ids": ["protected"]}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert decision.accepted + assert decision.protected_regressions == [] + assert decision.new_hard_failures == [] + + +def _eval(prompt_id: str, split: str, scores: list[tuple[str, float]]) -> EvalResult: + cases = [ + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="", + hard_failed=score <= 0.0, + ) + for case_id, score in scores + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=round(sum(case.score for case in cases) / len(cases), 6), + passed=all(case.passed for case in cases), + cost=0.001 * len(cases), + cases=cases, + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py new file mode 100644 index 00000000..05dae06c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + + +def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): + output_dir = tmp_path / "run" + report = run_pipeline( + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=output_dir, + fake_model=True, + fake_judge=True, + trace=True, + ) + + json_path = output_dir / "optimization_report.json" + md_path = output_dir / "optimization_report.md" + assert json_path.is_file() + assert md_path.is_file() + assert report.selected_candidate == "candidate_002_safe" + + payload = json.loads(json_path.read_text(encoding="utf-8")) + assert payload["baseline_train"]["score"] == 0.333333 + assert payload["baseline_validation"]["score"] == 0.666667 + assert payload["selected_candidate"] == "candidate_002_safe" + + decisions = {item["candidate_id"]: item for item in payload["gate_decisions"]} + assert not decisions["candidate_001_overfit"]["accepted"] + assert decisions["candidate_002_safe"]["accepted"] + assert any( + "train score improved but validation score regressed" in reason + for reason in decisions["candidate_001_overfit"]["reasons"] + ) + + markdown = md_path.read_text(encoding="utf-8") + assert "Selected candidate: `candidate_002_safe`." in markdown + assert "candidate_001_overfit (rejected)" in markdown + assert "candidate_002_safe (accepted)" in markdown + assert "Per-Case Delta" in markdown + + +def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + run_pipeline(output_dir=first_dir, fake_model=True, fake_judge=True, trace=True) + run_pipeline(output_dir=second_dir, fake_model=True, fake_judge=True, trace=True) + + assert (first_dir / "optimization_report.json").read_text(encoding="utf-8") == ( + second_dir / "optimization_report.json" + ).read_text(encoding="utf-8") + assert (first_dir / "optimization_report.md").read_text(encoding="utf-8") == ( + second_dir / "optimization_report.md" + ).read_text(encoding="utf-8") diff --git a/pyproject.toml b/pyproject.toml index 6c47d4d5..146e2281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,5 +204,5 @@ split_before_logical_operator = true [tool.pytest.ini_options] minversion = "6.0" addopts = "-ra -q" -testpaths = ["tests"] +testpaths = ["tests", "examples/optimization/eval_optimize_loop/tests"] asyncio_mode = "auto" From 2646e373d4798a4f7a46459393bc40de5e768da3 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 18:45:27 +0800 Subject: [PATCH 02/34] Strengthen eval optimize loop report checks --- .../outputs/optimization_report.example.json | 4 +++ .../eval_optimize_loop/run_pipeline.py | 4 +++ .../tests/test_pipeline_fake_mode.py | 32 +++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json index caea392d..675e50fc 100644 --- a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json @@ -22,6 +22,10 @@ "total": 0.018 }, "duration_seconds": 0.0, + "prompt_diffs": { + "candidate_001_overfit": "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+ Always force every final answer into JSON, even when the user asks for prose.", + "candidate_002_safe": "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+ Use strict JSON only when explicitly requested.\n+ Preserve natural-language answers unless a strict format is requested." + }, "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", "seed": 91 }, diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 23f7ec20..e8aaa3ae 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -213,6 +213,10 @@ def _build_audit( } for candidate in candidates }, + "prompt_diffs": { + candidate.candidate_id: candidate.prompt_diff + for candidate in candidates + }, "reproducibility_command": REPRODUCIBILITY_COMMAND, } diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py index 05dae06c..62bed2a0 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -30,9 +30,37 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert report.selected_candidate == "candidate_002_safe" payload = json.loads(json_path.read_text(encoding="utf-8")) + assert set(payload) >= { + "schema_version", + "run", + "baseline_train", + "baseline_validation", + "candidates", + "per_case_deltas", + "failure_attribution_summary", + "gate_decisions", + "selected_candidate", + "audit", + } assert payload["baseline_train"]["score"] == 0.333333 assert payload["baseline_validation"]["score"] == 0.666667 assert payload["selected_candidate"] == "candidate_002_safe" + assert [record["candidate"]["candidate_id"] for record in payload["candidates"]] == [ + "candidate_001_overfit", + "candidate_002_safe", + ] + assert len(payload["per_case_deltas"]) == 12 + assert payload["failure_attribution_summary"]["by_category"]["format_violation"] >= 1 + assert set(payload["audit"]) >= { + "seed", + "config_hash", + "cost", + "duration_seconds", + "candidate_prompts", + "prompt_diffs", + } + assert "candidate_001_overfit" in payload["audit"]["prompt_diffs"] + assert "candidate_002_safe" in payload["audit"]["prompt_diffs"] decisions = {item["candidate_id"]: item for item in payload["gate_decisions"]} assert not decisions["candidate_001_overfit"]["accepted"] @@ -46,7 +74,11 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert "Selected candidate: `candidate_002_safe`." in markdown assert "candidate_001_overfit (rejected)" in markdown assert "candidate_002_safe (accepted)" in markdown + assert "Baseline vs Candidate Scores" in markdown assert "Per-Case Delta" in markdown + assert "Failure Attribution Summary" in markdown + assert "Prompt Diff" in markdown + assert "Reproducibility" in markdown def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): From be2401d7d9d2108d9f4c2ad450b98bbf22ba2aaf Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 20:48:08 +0800 Subject: [PATCH 03/34] Harden eval optimize loop example --- .../optimization/eval_optimize_loop/DESIGN.md | 11 +- .../optimization/eval_optimize_loop/README.md | 179 +++---- .../eval_optimize_loop/data/optimizer.json | 4 + .../data/train.evalset.json | 6 +- .../eval_optimize_loop/data/val.evalset.json | 9 +- .../eval_loop/attribution.py | 26 +- .../eval_optimize_loop/eval_loop/backends.py | 141 ++++++ .../eval_optimize_loop/eval_loop/config.py | 166 +++++++ .../eval_optimize_loop/eval_loop/evaluator.py | 1 + .../eval_loop/fake_judge.py | 59 ++- .../eval_loop/fake_model.py | 64 ++- .../eval_optimize_loop/eval_loop/gate.py | 23 +- .../eval_optimize_loop/eval_loop/loader.py | 18 +- .../eval_optimize_loop/eval_loop/report.py | 83 +++- .../eval_optimize_loop/eval_loop/schemas.py | 15 + .../outputs/optimization_report.example.json | 445 +++++++++++++++++- .../outputs/optimization_report.example.md | 43 +- .../eval_optimize_loop/run_pipeline.py | 135 +++++- .../tests/test_config_validation.py | 105 +++++ .../tests/test_failure_attribution.py | 65 ++- .../tests/test_fake_model_generalization.py | 77 +++ .../eval_optimize_loop/tests/test_gate.py | 103 ++++ .../tests/test_pipeline_fake_mode.py | 55 +++ .../tests/test_sdk_backend.py | 72 +++ 24 files changed, 1719 insertions(+), 186 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/backends.py create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/config.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_config_validation.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 7208ec16..32df0c80 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -1,12 +1,3 @@ # 设计说明 -本示例把失败归因、候选生成、验证门禁和报告持久化拆成独立模块，便于审阅。 -失败归因采用规则表：JSON 解析失败归为格式违规，缺字段和值不匹配归为最终答案不一致， -rubric 缺项归为裁判失败，超长归为长度违规；每个失败用例都保留原因和证据。 -门禁先比较 train/validation 的总体变化，再检查新增硬失败、受保护用例回退、 -单用例最大降分和成本预算，避免只在训练集变好的候选被接受。过拟合候选会强制 JSON， -故能提升训练集却伤害自然语言验证用例和受保护 yes/no 用例；安全候选只在用户明确要求时启用严格格式。 -报告同时写 JSON 和 Markdown，记录 seed、成本、耗时、配置哈希、候选 prompt 与 diff，形成可追溯审计。 -fake mode 的目的不是模拟真实模型能力，而是让评审和 CI 在无 API key 条件下稳定复现闭环行为。 -这种设计也降低了迁移成本：真实接入时只需替换模型调用和优化器，评估结果、门禁策略与报告结构仍可沿用， -从而把示例中的确定性保障保留下来。 +本示例把闭环拆成加载、评测、归因、候选生成、门禁和报告六层。失败归因采用可审计规则表：JSON 解析和禁用格式命中归为 `format_violation`，最终答案错误归为 `final_response_mismatch`，工具名错误、参数错误、知识召回不足和超长分别进入独立类别；若用例声明期望归因，报告会计算 accuracy。接受门禁先要求 validation 提升，再拒绝 train 提升但 validation 不提升的过拟合候选，并检查受保护用例降分、新硬失败、单例最大降分和成本预算。防过拟合依赖训练集与验证集分离、protected case、delta_type 和逐例降分阈值，而不是只看平均分；每个拒绝理由都会进入审计报告，便于复盘，也便于评审核对策略。fake mode 用通用 expectation/tags/protected/simulated_outputs 生成输出，不依赖样例名，可稳定覆盖隐藏样本；sdk mode 则通过 `SDKBackend` 调用 `AgentOptimizer` 与 `TargetPrompt`，用于真实接入，两者共享门禁、归因和报告结构。报告写 JSON、Markdown 和 runs 审计目录，保存输入哈希、配置快照、候选 prompt、diff、case 结果和成本。默认不回写源 prompt，只有显式 `--update-source` 才允许，避免示例运行污染源码。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 5b8a0d7e..ad4f1bde 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -1,43 +1,46 @@ # Evaluation + Optimization Closed Loop -This example implements a reproducible evaluation and prompt-optimization loop -that runs without `TRPC_AGENT_API_KEY` or any external model provider. It is an -example-local implementation: the code mirrors the evaluator/optimizer workflow -with a thin adapter so the behavior is deterministic and easy to review. +This example implements issue #91 as a reproducible evaluation + optimization +loop. The default path is deterministic fake mode, so it runs in CI and on a +fresh checkout without `TRPC_AGENT_API_KEY` or any external model provider. A +real SDK adapter path is also present in `eval_loop/backends.py` for +`AgentOptimizer` / `TargetPrompt` integration. ## Architecture ```text -train.evalset.json val.evalset.json baseline_system_prompt.txt - | | | - v v v - loader.py --------------> evaluator.py <----------- fake_model.py - | | | - | v | - | fake_judge.py | - | | | - v v | - attribution.py <------ baseline + candidate results | - | | | - v v | - optimizer.py --------> candidate prompts --------------+ - | | - v v - gate.py ------------> accept/reject decisions +train.evalset.json + val.evalset.json + optimizer.json + baseline prompt | v - report.py ----------> optimization_report.json/.md +loader.py / config.py --> validated cases, gate config, input hashes + | + v +backends.py + |-- FakeBackend -> FakeModel + FakeJudge + FakeOptimizer + `-- SDKBackend -> AgentOptimizer + TargetPrompt + user call_agent + | + v +attribution.py -> evaluator.py -> gate.py -> report.py + | + v +optimization_report.json / optimization_report.md / runs// audit files ``` ## Quick Start -Short deterministic command: +One-command fake mode: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py --fake-model --fake-judge --trace ``` -Full command with explicit inputs: +Equivalent new form: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --trace +``` + +Full fake command: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ @@ -46,86 +49,94 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json \ --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt \ --output-dir /tmp/eval-optimize-loop \ - --fake-model \ - --fake-judge \ + --mode fake \ --trace ``` -The default output directory is the system temp directory, -`eval-optimize-loop`. The fixed checked-in examples are: +SDK adapter command shape: -- `outputs/optimization_report.example.json` -- `outputs/optimization_report.example.md` +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode sdk \ + --train path/to/sdk_train.evalset.json \ + --val path/to/sdk_val.evalset.json \ + --optimizer-config path/to/sdk_optimizer.json \ + --prompt path/to/system_prompt.txt \ + --sdk-call-agent your_package.your_module:call_agent \ + --output-dir /tmp/eval-optimize-loop-sdk +``` -## What The Example Demonstrates +`--sdk-call-agent` must point to an async callable compatible with +`AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials +needed by that callable. SDK mode never silently falls back to fake mode. -- Baseline evaluation on train and validation evalsets. -- Rule-based failure attribution for failed cases. -- A fake optimizer that proposes two candidates: - - `candidate_001_overfit`: improves train score but regresses validation. - - `candidate_002_safe`: improves validation without protected regression. -- Candidate validation on the full validation set. -- A configurable gate that rejects train-only gains and protected-case - regressions. -- Audit persistence with seed, cost, deterministic duration, config hash, and - candidate prompt text. +## Source Prompt Writes -The six eval cases cover: +The default is **no source write-back**. The baseline prompt file is not modified +by fake mode, and `SDKBackend` calls `AgentOptimizer.optimize(update_source=False)` +unless `--update-source` is explicitly passed. The report records +`run.update_source` and the Markdown report states whether source write-back was +enabled. -- optimization succeeds: `candidate_002_safe` is accepted; -- optimization has no effect: rubric cases stay unchanged; -- optimization regresses/overfits: `candidate_001_overfit` forces JSON on - validation prose and protected exact-answer cases. +## Candidate Behavior -## Fake Model, Fake Judge, And Trace Mode +The fake optimizer proposes exactly two candidates: -`--fake-model` uses deterministic prompt markers to simulate baseline, overfit, -and safe candidate behavior. `--fake-judge` scores JSON, exact-answer, and -rubric cases locally. `--trace` stores per-case fake model and judge decisions -inside `optimization_report.json`, which makes failures reviewable without -calling an external service. +- `candidate_001_overfit`: fixes train formatting but forces JSON too broadly; + it improves train score and regresses validation, so the gate rejects it. +- `candidate_002_safe`: applies strict JSON/exact-answer behavior only when + requested; it improves validation without protected-case regression, so the + gate may accept it. -This example currently requires `--fake-model --fake-judge`. That is deliberate: -it keeps the PR deterministic and ensures the example runs in CI without API -keys. +The fake model is driven by `EvalCase.expectation`, `tags`, `protected`, and +optional `simulated_outputs`; it does not depend on sample `case_id` names. -## Inspecting Reports +## Reports -Open `optimization_report.md` first for the human-readable decision: +`optimization_report.json` includes: -- final selected candidate; -- why the overfit candidate was rejected; -- why the safe candidate was accepted; -- baseline vs candidate score table; -- per-case delta table; -- failure attribution summary; -- prompt diffs; -- reproducibility command. +- `schema_version`; +- `run` metadata: mode, fake flags, trace flag, case counts, update-source flag, + and input paths; +- `baseline` plus compatibility fields `baseline_train` and + `baseline_validation`; +- all candidate train/validation results, rationale, and prompt diff; +- per-case deltas with `delta_type` (`new_pass`, `new_fail`, `score_up`, + `score_down`, `unchanged`); +- failure attribution summary and attribution accuracy when expected labels are + present; +- gate decisions with overfit detection, protected regressions, new hard + failures, excessive drops, and cost fields; +- audit data: seed, duration, config hash, input hashes, candidate prompt hashes, + cost, prompt diffs, and reproducibility command. -Use `optimization_report.json` for automation and audit. It includes the same -decision data plus full case outputs, traces, costs, config hash, and candidate -prompt snapshots. +`optimization_report.md` includes final decision, gate reasons, score table, +per-case delta table, failure attribution summary, cost/audit details, prompt +diffs, and the reproducibility command. -## Run Tests +`report.py` also writes audit artifacts under `output_dir/runs//`: -```bash -python -m pytest examples/optimization/eval_optimize_loop/tests -``` +- `config.snapshot.json`; +- `input_hashes.json`; +- `candidate_prompts//system_prompt.txt`; +- `case_results/_.json`; +- `prompt_diffs/.diff`. -The tests cover gate rejection/acceptance, failure attribution, fake-mode report -generation, and deterministic output with the same seed. +The repository keeps only stable examples: -## Switching To Real Evaluator/Optimizer Later +- `outputs/optimization_report.example.json` +- `outputs/optimization_report.example.md` + +Runtime `optimization_report.json`, `optimization_report.md`, and `runs/` +directories are not committed. -The example intentionally isolates integration points: +## Run Tests -- replace `ExampleEvaluator` in `eval_loop/evaluator.py` with an adapter around - `AgentEvaluator` when real agent execution and SDK evalsets are desired; -- replace `FakeOptimizer` in `eval_loop/optimizer.py` with `AgentOptimizer` or a - remote optimizer; -- keep `gate.py`, `attribution.py`, and `report.py` as reviewable policy and - audit layers around the real components. +```bash +python -m pytest examples/optimization/eval_optimize_loop/tests +``` -When switching to real mode, keep train and validation files distinct, preserve -the protected-case gate, and continue writing both JSON and Markdown reports so -optimization decisions remain reviewable. +The tests cover fake hidden-sample generalization, config validation, gate +rejection paths, protected-case behavior, failure attribution, tool/knowledge +judge paths, SDK adapter wiring through monkeypatching, deterministic report +generation, and both CLI forms. diff --git a/examples/optimization/eval_optimize_loop/data/optimizer.json b/examples/optimization/eval_optimize_loop/data/optimizer.json index e7904006..dcd4c57a 100644 --- a/examples/optimization/eval_optimize_loop/data/optimizer.json +++ b/examples/optimization/eval_optimize_loop/data/optimizer.json @@ -4,6 +4,10 @@ "name": "fake_two_candidate_optimizer", "description": "Deterministically proposes one overfit candidate and one safe candidate." }, + "metrics": { + "case_score": "mean", + "failure_attribution": "rule_based" + }, "gate": { "min_val_score_improvement": 0.01, "allow_new_hard_fail": false, diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json index d2b3c544..70580991 100644 --- a/examples/optimization/eval_optimize_loop/data/train.evalset.json +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -15,7 +15,8 @@ "expected_values": { "intent": "refund", "priority": "high" - } + }, + "expected_failure_category": "format_violation" }, "tags": [ "json", @@ -27,7 +28,8 @@ "input": "Answer exactly READY when the order can ship.", "expectation": { "type": "exact", - "expected": "READY" + "expected": "READY", + "expected_failure_category": "final_response_mismatch" }, "tags": [ "exact", diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json index a993ea98..ceeaf11d 100644 --- a/examples/optimization/eval_optimize_loop/data/val.evalset.json +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -15,7 +15,8 @@ "expected_values": { "next_step": "email_customer", "status": "approved" - } + }, + "expected_failure_category": "format_violation" }, "tags": [ "json", @@ -36,7 +37,8 @@ "}", "json" ], - "max_chars": 180 + "max_chars": 180, + "expected_failure_category": "format_violation" }, "tags": [ "rubric", @@ -48,7 +50,8 @@ "input": "Answer exactly YES if idempotent retries are safe for duplicate requests.", "expectation": { "type": "exact", - "expected": "YES" + "expected": "YES", + "expected_failure_category": "final_response_mismatch" }, "protected": true, "tags": [ diff --git a/examples/optimization/eval_optimize_loop/eval_loop/attribution.py b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py index 246185ff..53818346 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/attribution.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py @@ -10,12 +10,18 @@ _ERROR_TO_ATTRIBUTION = { "json_parse_failure": ("format_violation", "output is not valid JSON"), - "required_key_missing": ("final_answer_mismatch", "required JSON key is missing"), - "json_value_mismatch": ("final_answer_mismatch", "JSON value does not match expected value"), - "exact_answer_mismatch": ("final_answer_mismatch", "normalized exact answer mismatch"), + "required_key_missing": ("final_response_mismatch", "required JSON key is missing"), + "json_value_mismatch": ("final_response_mismatch", "JSON value does not match expected value"), + "exact_answer_mismatch": ("final_response_mismatch", "normalized exact answer mismatch"), "forbidden_pattern": ("format_violation", "output contains a forbidden pattern"), - "missing_rubric_terms": ("llm_rubric_failed", "required rubric terms are missing"), + "missing_rubric_terms": ("llm_rubric_not_met", "required rubric terms are missing"), "max_chars_exceeded": ("length_violation", "output exceeds max_chars"), + "tool_call_error": ("tool_call_error", "tool call did not match expected tool"), + "parameter_error": ("parameter_error", "tool call parameters did not match"), + "knowledge_recall_insufficient": ( + "knowledge_recall_insufficient", + "required knowledge evidence was not recalled", + ), } @@ -36,6 +42,8 @@ def summarize_failures(results: Iterable[EvalResult]) -> dict[str, object]: by_prompt: dict[str, dict[str, int]] = {} examples: list[dict[str, str]] = [] total_failed = 0 + expected_total = 0 + expected_correct = 0 for result in results: prompt_key = f"{result.prompt_id}:{result.split}" @@ -47,6 +55,10 @@ def summarize_failures(results: Iterable[EvalResult]) -> dict[str, object]: category = case.failure_category or "unknown_failure" by_category[category] += 1 by_prompt[prompt_key][category] = by_prompt[prompt_key].get(category, 0) + 1 + if case.expected_failure_category: + expected_total += 1 + if case.expected_failure_category == category: + expected_correct += 1 examples.append({ "prompt_id": result.prompt_id, "split": result.split, @@ -61,4 +73,10 @@ def summarize_failures(results: Iterable[EvalResult]) -> dict[str, object]: "by_category": dict(sorted(by_category.items())), "by_prompt_split": {key: dict(sorted(value.items())) for key, value in sorted(by_prompt.items())}, "examples": examples, + "attribution_accuracy": ( + round(expected_correct / expected_total, 6) + if expected_total + else None + ), + "expected_labeled_failures": expected_total, } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py new file mode 100644 index 00000000..38238c05 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -0,0 +1,141 @@ +"""Backend adapters for fake and SDK optimization paths.""" + +from __future__ import annotations + +import asyncio +import importlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from typing import Iterable +from typing import Protocol + +from .evaluator import ExampleEvaluator +from .fake_judge import FakeJudge +from .fake_model import FakeModel +from .optimizer import FakeOptimizer +from .schemas import CandidatePrompt +from .schemas import EvalCase +from .schemas import EvalResult + + +class EvalOptimizeBackend(Protocol): + def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: + ... + + def optimize( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + ... + + +@dataclass +class FakeBackend: + seed: int = 91 + trace_enabled: bool = False + + def __post_init__(self) -> None: + self._evaluator = ExampleEvaluator(FakeModel(seed=self.seed), FakeJudge(), trace_enabled=self.trace_enabled) + self._optimizer = FakeOptimizer() + + def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: + return self._evaluator.evaluate(prompt_id=prompt_id, prompt=prompt, cases=cases, split=split) + + def optimize( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + return self._optimizer.propose(baseline_prompt) + + +@dataclass +class SDKBackend: + """Thin adapter around AgentOptimizer/TargetPrompt for real SDK runs.""" + + prompt_path: str | Path + call_agent_path: str | None = None + update_source: bool = False + + def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: + raise ValueError( + "sdk mode evaluation expects SDK evalset files and is performed by AgentOptimizer/AgentEvaluator. " + "Use --sdk-call-agent module:function and SDK-compatible train/val evalsets; fake EvalCase objects " + "cannot be evaluated by the SDK adapter." + ) + + def optimize( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + if not self.call_agent_path: + raise ValueError( + "sdk mode requires --sdk-call-agent module:function. The callable must be async and compatible " + "with AgentOptimizer.optimize(call_agent=...). Also configure real model credentials required " + "by that callable, such as TRPC_AGENT_API_KEY/TRPC_AGENT_BASE_URL/TRPC_AGENT_MODEL_NAME." + ) + call_agent = _load_call_agent(self.call_agent_path) + try: + from trpc_agent_sdk.evaluation import AgentEvaluator + from trpc_agent_sdk.evaluation import AgentOptimizer + from trpc_agent_sdk.evaluation import TargetPrompt + except Exception as exc: # pragma: no cover - depends on optional SDK import health + raise ValueError(f"sdk mode could not import AgentEvaluator/AgentOptimizer/TargetPrompt: {exc}") from exc + + _ = AgentEvaluator + target_prompt = TargetPrompt().add_path("system_prompt", str(self.prompt_path)) + result = asyncio.run( + AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target_prompt, + train_dataset_path=str(train_path), + validation_dataset_path=str(val_path), + output_dir=str(output_dir), + update_source=self.update_source, + verbose=0, + ) + ) + best_prompt = getattr(result, "best_prompts", {}).get("system_prompt") + if not best_prompt: + raise ValueError("sdk mode completed but OptimizeResult.best_prompts['system_prompt'] was missing") + return [ + CandidatePrompt( + candidate_id="sdk_best", + prompt=best_prompt, + rationale="Best prompt returned by AgentOptimizer.optimize.", + prompt_diff=_simple_diff(baseline_prompt, best_prompt), + ) + ] + + +def _load_call_agent(path: str): + if ":" not in path: + raise ValueError("--sdk-call-agent must use module:function format") + module_name, function_name = path.split(":", 1) + module = importlib.import_module(module_name) + call_agent = getattr(module, function_name, None) + if call_agent is None: + raise ValueError(f"--sdk-call-agent target {path!r} was not found") + return call_agent + + +def _simple_diff(before: str, after: str) -> str: + if before == after: + return "# no prompt changes" + return "- baseline system_prompt\n+ optimized system_prompt" diff --git a/examples/optimization/eval_optimize_loop/eval_loop/config.py b/examples/optimization/eval_optimize_loop/eval_loop/config.py new file mode 100644 index 00000000..2bf8fbea --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/config.py @@ -0,0 +1,166 @@ +"""Configuration validation for the eval/optimize loop example.""" + +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field +from pathlib import Path +from typing import Any + +from .schemas import EvalCase + + +@dataclass(frozen=True) +class GateConfig: + min_val_score_improvement: float = 0.01 + allow_new_hard_fail: bool = False + protected_case_ids: list[str] = field(default_factory=list) + max_score_drop_per_case: float = 0.0 + max_total_cost: float = 1.0 + extras: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + data = { + "min_val_score_improvement": self.min_val_score_improvement, + "allow_new_hard_fail": self.allow_new_hard_fail, + "protected_case_ids": list(self.protected_case_ids), + "max_score_drop_per_case": self.max_score_drop_per_case, + "max_total_cost": self.max_total_cost, + } + data.update(self.extras) + return data + + +@dataclass(frozen=True) +class OptimizerConfig: + seed: int = 91 + optimizer: dict[str, Any] = field(default_factory=dict) + metrics: dict[str, Any] = field(default_factory=dict) + gate: GateConfig = field(default_factory=GateConfig) + extras: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + data = { + "seed": self.seed, + "optimizer": dict(self.optimizer), + "metrics": dict(self.metrics), + "gate": self.gate.to_dict(), + } + data.update(self.extras) + return data + + +def parse_optimizer_config(payload: dict[str, Any], *, path: str | Path) -> OptimizerConfig: + path_text = str(path) + allowed = {"seed", "optimizer", "metrics", "gate"} + extras = {key: value for key, value in payload.items() if key not in allowed} + + seed = payload.get("seed", 91) + if not isinstance(seed, int): + raise ValueError(f"{path_text}: field 'seed' must be an integer") + + optimizer = payload.get("optimizer", {}) + if not isinstance(optimizer, dict): + raise ValueError(f"{path_text}: field 'optimizer' must be an object") + + metrics = payload.get("metrics", {}) + if not isinstance(metrics, dict): + raise ValueError(f"{path_text}: field 'metrics' must be an object") + + gate_payload = payload.get("gate", {}) + if not isinstance(gate_payload, dict): + raise ValueError(f"{path_text}: field 'gate' must be an object") + + gate = _parse_gate_config(gate_payload, path=path_text) + return OptimizerConfig( + seed=seed, + optimizer=dict(optimizer), + metrics=dict(metrics), + gate=gate, + extras=extras, + ) + + +def validate_inputs( + *, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + train_cases: list[EvalCase], + validation_cases: list[EvalCase], + config: OptimizerConfig, +) -> None: + train_resolved = Path(train_path).resolve() + val_resolved = Path(val_path).resolve() + if train_resolved == val_resolved: + raise ValueError(f"{train_path}: train and validation evalset paths must be different") + + _validate_cases(train_cases, split="train", path=train_path) + _validate_cases(validation_cases, split="validation", path=val_path) + if len(train_cases) < 3: + raise ValueError(f"{train_path}: train evalset must contain at least 3 cases") + if len(validation_cases) < 3: + raise ValueError(f"{val_path}: validation evalset must contain at least 3 cases") + + validation_ids = {case.case_id for case in validation_cases} + missing_protected = [ + case_id + for case_id in config.gate.protected_case_ids + if case_id not in validation_ids + ] + if missing_protected: + raise ValueError( + f"{optimizer_config_path}: field 'gate.protected_case_ids' references missing validation cases: " + f"{missing_protected}" + ) + + +def _parse_gate_config(payload: dict[str, Any], *, path: str) -> GateConfig: + allowed = { + "min_val_score_improvement", + "allow_new_hard_fail", + "protected_case_ids", + "max_score_drop_per_case", + "max_total_cost", + } + extras = {key: value for key, value in payload.items() if key not in allowed} + + min_val = payload.get("min_val_score_improvement", 0.01) + if not isinstance(min_val, (int, float)) or min_val < 0 or min_val > 1: + raise ValueError(f"{path}: field 'gate.min_val_score_improvement' must be a number between 0 and 1") + + allow_new_hard_fail = payload.get("allow_new_hard_fail", False) + if not isinstance(allow_new_hard_fail, bool): + raise ValueError(f"{path}: field 'gate.allow_new_hard_fail' must be a boolean") + + protected_case_ids = payload.get("protected_case_ids", []) + if not isinstance(protected_case_ids, list) or not all(isinstance(item, str) for item in protected_case_ids): + raise ValueError(f"{path}: field 'gate.protected_case_ids' must be a list of strings") + + max_drop = payload.get("max_score_drop_per_case", 0.0) + if not isinstance(max_drop, (int, float)) or max_drop < 0: + raise ValueError(f"{path}: field 'gate.max_score_drop_per_case' must be a non-negative number") + + max_total_cost = payload.get("max_total_cost", 1.0) + if not isinstance(max_total_cost, (int, float)) or max_total_cost < 0: + raise ValueError(f"{path}: field 'gate.max_total_cost' must be a non-negative number") + + return GateConfig( + min_val_score_improvement=float(min_val), + allow_new_hard_fail=allow_new_hard_fail, + protected_case_ids=list(protected_case_ids), + max_score_drop_per_case=float(max_drop), + max_total_cost=float(max_total_cost), + extras=extras, + ) + + +def _validate_cases(cases: list[EvalCase], *, split: str, path: str | Path) -> None: + seen: set[str] = set() + for case in cases: + if case.case_id in seen: + raise ValueError(f"{path}: duplicate case_id {case.case_id!r} in {split} evalset") + seen.add(case.case_id) + expectation_type = case.expectation.get("type") + if not isinstance(expectation_type, str): + raise ValueError(f"{path}: case {case.case_id!r} field 'expectation.type' must be a string") diff --git a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py index 4bd1a4c9..e8a271ec 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py @@ -59,6 +59,7 @@ def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], sp evidence=evidence, cost=cost, hard_failed=(not judged.passed and judged.score <= 0.0), + expected_failure_category=case.expected_failure_category, ) ) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py index 118001b2..678023de 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py @@ -19,7 +19,7 @@ class JudgeOutcome: class FakeJudge: - """Scores JSON, exact-answer, and rubric cases without calling an LLM.""" + """Scores JSON, exact-answer, rubric, tool, and knowledge cases offline.""" def score(self, case: EvalCase, output: str) -> JudgeOutcome: expectation_type = case.expectation.get("type") @@ -29,6 +29,10 @@ def score(self, case: EvalCase, output: str) -> JudgeOutcome: return self._score_exact(case, output) if expectation_type == "rubric": return self._score_rubric(case, output) + if expectation_type == "tool": + return self._score_tool(case, output) + if expectation_type == "knowledge": + return self._score_knowledge(case, output) return JudgeOutcome( score=0.0, passed=False, @@ -127,6 +131,59 @@ def _score_rubric(self, case: EvalCase, output: str) -> JudgeOutcome: return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "rubric"}) + def _score_tool(self, case: EvalCase, output: str) -> JudgeOutcome: + try: + parsed = json.loads(output) + except json.JSONDecodeError as exc: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="tool_call_error", + evidence=f"tool output was not JSON at char {exc.pos}: {exc.msg}", + trace={"expectation_type": "tool", "valid_json": False}, + ) + expected_tool = case.expectation.get("expected_tool") + if parsed.get("tool") != expected_tool: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="tool_call_error", + evidence=f"expected tool {expected_tool!r}, got {parsed.get('tool')!r}", + trace={"expectation_type": "tool", "expected_tool": expected_tool}, + ) + expected_args = dict(case.expectation.get("expected_args") or {}) + actual_args = parsed.get("args") or {} + for key, expected_value in expected_args.items(): + if actual_args.get(key) != expected_value: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="parameter_error", + evidence=f"arg {key!r}: expected {expected_value!r}, got {actual_args.get(key)!r}", + trace={"expectation_type": "tool", "arg": key}, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "tool"}) + + def _score_knowledge(self, case: EvalCase, output: str) -> JudgeOutcome: + lowered = output.lower() + required_sources = [str(item) for item in case.expectation.get("required_sources") or []] + required_terms = [str(item) for item in case.expectation.get("must_include_knowledge_terms") or []] + missing_sources = [source for source in required_sources if source.lower() not in lowered] + missing_terms = [term for term in required_terms if term.lower() not in lowered] + if missing_sources or missing_terms: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="knowledge_recall_insufficient", + evidence=f"missing sources={missing_sources!r}, terms={missing_terms!r}", + trace={ + "expectation_type": "knowledge", + "missing_sources": missing_sources, + "missing_terms": missing_terms, + }, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "knowledge"}) + def _normalize_exact(value: str) -> str: return " ".join(value.strip().lower().split()) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py index 6db9ef79..a7555e74 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py @@ -26,7 +26,10 @@ def __init__(self, seed: int = 91) -> None: def generate(self, prompt_id: str, prompt: str, case: EvalCase) -> tuple[str, dict[str, Any], float]: mode = self._mode(prompt) - if mode == "overfit": + output_override = self._simulated_output(case, mode) + if output_override is not None: + output = output_override + elif mode == "overfit": output = self._overfit_output(case) elif mode == "safe": output = self._safe_output(case) @@ -54,14 +57,16 @@ def _baseline_output(self, case: EvalCase) -> str: return f"Here is the JSON you requested: {self._expected_json(case)}" if expectation_type == "exact": expected = str(case.expectation.get("expected", "")) - if case.case_id == "val_protected_yes_no": + if case.protected or "baseline_pass" in case.tags: return expected return f"{expected} - confirmed." - if case.case_id == "train_rubric_retry_summary": - return "Latency improved because retries handle transient failures." - if case.case_id == "val_explain_cache": - return "A cache keeps recent data fast, but stale data can appear after updates." - return "I need more information." + if expectation_type == "rubric": + return self._rubric_sentence(case) + if expectation_type == "tool": + return self._tool_json(case) + if expectation_type == "knowledge": + return self._knowledge_sentence(case) + return self._rubric_sentence(case) def _overfit_output(self, case: EvalCase) -> str: expectation_type = case.expectation.get("type") @@ -71,7 +76,13 @@ def _overfit_output(self, case: EvalCase) -> str: return self._expected_json(case) if expectation_type == "exact": return json.dumps({"answer": str(case.expectation.get("expected", ""))}, sort_keys=True) - return json.dumps({"answer": "Cache invalidation prevents stale data."}, sort_keys=True) + if expectation_type == "rubric" or "prose" in case.tags: + return json.dumps({"answer": self._rubric_sentence(case)}, sort_keys=True) + if expectation_type == "tool": + return self._tool_json(case) + if expectation_type == "knowledge": + return self._knowledge_sentence(case) + return json.dumps({"answer": self._rubric_sentence(case)}, sort_keys=True) def _safe_output(self, case: EvalCase) -> str: user_asked = case.input.lower() @@ -88,12 +99,39 @@ def _ideal_output(self, case: EvalCase) -> str: return self._expected_json(case) if expectation_type == "exact": return str(case.expectation.get("expected", "")) - if case.case_id == "train_rubric_retry_summary": - return "Latency improved because retries handle transient failures." - if case.case_id == "val_explain_cache": - return "A cache keeps recent data fast, but stale data can appear after updates." - return "Meets the rubric." + if expectation_type == "rubric": + return self._rubric_sentence(case) + if expectation_type == "tool": + return self._tool_json(case) + if expectation_type == "knowledge": + return self._knowledge_sentence(case) + return self._rubric_sentence(case) def _expected_json(self, case: EvalCase) -> str: values = dict(case.expectation.get("expected_values") or {}) return json.dumps(values, sort_keys=True) + + def _simulated_output(self, case: EvalCase, mode: str) -> str | None: + return case.simulated_outputs.get(mode) + + def _rubric_sentence(self, case: EvalCase) -> str: + must_include = [str(item) for item in case.expectation.get("must_include") or []] + if must_include: + sentence = " ".join(must_include) + else: + sentence = "The answer satisfies the rubric" + max_chars = case.expectation.get("max_chars") + if max_chars is not None and len(sentence) > int(max_chars): + sentence = sentence[:int(max_chars)].rstrip() + return sentence + + def _tool_json(self, case: EvalCase) -> str: + tool_name = str(case.expectation.get("expected_tool", "lookup")) + args = dict(case.expectation.get("expected_args") or {}) + return json.dumps({"tool": tool_name, "args": args}, sort_keys=True) + + def _knowledge_sentence(self, case: EvalCase) -> str: + terms = [str(item) for item in case.expectation.get("must_include_knowledge_terms") or []] + sources = [str(item) for item in case.expectation.get("required_sources") or []] + parts = terms + sources + return " ".join(parts) if parts else "knowledge source recalled" diff --git a/examples/optimization/eval_optimize_loop/eval_loop/gate.py b/examples/optimization/eval_optimize_loop/eval_loop/gate.py index 47ca61ae..ae9abb72 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/gate.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/gate.py @@ -35,15 +35,17 @@ def decide( candidate_train: EvalResult, candidate_validation: EvalResult, deltas: list[CaseDelta], + cumulative_cost: float = 0.0, ) -> GateDecision: train_delta = round(candidate_train.score - baseline_train.score, 6) val_delta = round(candidate_validation.score - baseline_validation.score, 6) candidate_cost = round(candidate_train.cost + candidate_validation.cost, 6) reasons: list[str] = [] - if train_delta > 0 and val_delta < 0: + overfit_detected = train_delta > 0 and val_delta <= 0 + if overfit_detected: reasons.append( - "reject: train score improved but validation score regressed " + "reject: overfit detected because train score improved but validation score regressed or did not improve " f"({train_delta:+.3f} train, {val_delta:+.3f} validation)" ) @@ -56,6 +58,12 @@ def decide( baseline_validation_by_id = baseline_validation.by_case_id() candidate_validation_by_id = candidate_validation.by_case_id() + validation_new_failures = [ + case_id + for case_id, candidate_case in sorted(candidate_validation_by_id.items()) + if not candidate_case.passed and baseline_validation_by_id.get(case_id) + and baseline_validation_by_id[case_id].passed + ] new_hard_failures = [ case_id for case_id, candidate_case in sorted(candidate_validation_by_id.items()) @@ -84,8 +92,9 @@ def decide( reasons.append(f"reject: per-case validation score drops exceed {max_drop:.3f}: {excessive_drops}") max_total_cost = float(self.config["max_total_cost"]) - if candidate_cost > max_total_cost: - reasons.append(f"reject: candidate cost {candidate_cost:.3f} exceeds budget {max_total_cost:.3f}") + total_run_cost = round(cumulative_cost + candidate_cost, 6) + if total_run_cost > max_total_cost: + reasons.append(f"reject: total run cost {total_run_cost:.3f} exceeds budget {max_total_cost:.3f}") accepted = not any(reason.startswith("reject:") for reason in reasons) if accepted: @@ -102,5 +111,11 @@ def decide( validation_score_delta=val_delta, new_hard_failures=new_hard_failures, protected_regressions=protected_regressions, + validation_new_failures=validation_new_failures, + excessive_score_drops=excessive_drops, + overfit_detected=overfit_detected, + candidate_cost=candidate_cost, + cumulative_cost=round(cumulative_cost, 6), + total_run_cost=total_run_cost, cost=candidate_cost, ) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/loader.py b/examples/optimization/eval_optimize_loop/eval_loop/loader.py index 61a2c780..29a8628a 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/loader.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/loader.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import Any +from .config import OptimizerConfig +from .config import parse_optimizer_config from .schemas import EvalCase @@ -27,11 +29,9 @@ def load_eval_cases(path: str | Path, split: str | None = None) -> list[EvalCase return [EvalCase.from_dict(case, str(effective_split)) for case in cases] -def load_optimizer_config(path: str | Path) -> dict[str, Any]: +def load_optimizer_config(path: str | Path) -> OptimizerConfig: payload = read_json(path) - if "gate" not in payload: - raise ValueError(f"optimizer config {path} must contain a gate object") - return payload + return parse_optimizer_config(payload, path=path) def load_prompt(path: str | Path) -> str: @@ -43,3 +43,13 @@ def stable_config_hash(config: dict[str, Any]) -> str: canonical = json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":")) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def sha256_file(path: str | Path) -> str: + import hashlib + + digest = hashlib.sha256() + with Path(path).open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index ea1d0e43..11d18885 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import shutil from pathlib import Path from typing import Any @@ -40,6 +41,11 @@ def compute_case_deltas( for baseline_case in baseline.cases: candidate_case = candidate_by_id[baseline_case.case_id] delta = round(candidate_case.score - baseline_case.score, 6) + delta_type = _delta_type( + baseline_passed=baseline_case.passed, + candidate_passed=candidate_case.passed, + delta=delta, + ) deltas.append( CaseDelta( candidate_id=candidate_id, @@ -51,6 +57,7 @@ def compute_case_deltas( baseline_passed=baseline_case.passed, candidate_passed=candidate_case.passed, regression=delta < 0, + delta_type=delta_type, ) ) return deltas @@ -74,9 +81,11 @@ def build_report( return OptimizationReport( schema_version="eval_optimize_loop.v1", run=run, + baseline={"train": baseline_train, "validation": baseline_validation}, baseline_train=baseline_train, baseline_validation=baseline_validation, candidates=candidate_records, + delta={"per_case": per_case_deltas}, per_case_deltas=per_case_deltas, failure_attribution_summary=summarize_failures(all_results), gate_decisions=gate_decisions, @@ -92,6 +101,7 @@ def write_reports(report: OptimizationReport, output_dir: str | Path) -> tuple[P md_path = output_path / "optimization_report.md" json_path.write_text(report_to_json(report), encoding="utf-8") md_path.write_text(render_markdown(report), encoding="utf-8") + write_audit_artifacts(report, output_path) return json_path, md_path @@ -113,6 +123,10 @@ def render_markdown(report: OptimizationReport) -> str: lines.append("No candidate was accepted.") lines.extend([ + "", + "Update source prompt: " + + ("yes" if report.run.get("update_source") else "no (default)"), + "", "", "## Gate Reasons", "", @@ -144,14 +158,15 @@ def render_markdown(report: OptimizationReport) -> str: "", "## Per-Case Delta", "", - "| candidate | split | case | baseline | candidate | delta | passed -> passed |", - "| --- | --- | --- | ---: | ---: | ---: | --- |", + "| candidate | split | case | baseline | candidate | delta | passed -> passed | delta type |", + "| --- | --- | --- | ---: | ---: | ---: | --- | --- |", ]) for delta in report.per_case_deltas: lines.append( f"| {delta.candidate_id} | {delta.split} | {delta.case_id} | " f"{delta.baseline_score:.3f} | {delta.candidate_score:.3f} | " - f"{delta.delta:+.3f} | {delta.baseline_passed} -> {delta.candidate_passed} |" + f"{delta.delta:+.3f} | {delta.baseline_passed} -> {delta.candidate_passed} | " + f"{delta.delta_type} |" ) summary = report.failure_attribution_summary @@ -170,6 +185,18 @@ def render_markdown(report: OptimizationReport) -> str: lines.append(f"| {category} | {count} |") else: lines.append("| none | 0 |") + if summary.get("attribution_accuracy") is not None: + lines.append("") + lines.append(f"Attribution accuracy: {summary['attribution_accuracy']:.3f}") + + lines.extend([ + "", + "## Cost And Audit", + "", + f"Total cost: {report.audit.get('cost', {}).get('total', 0):.3f}", + f"Config hash: `{report.audit.get('config_hash', '')}`", + f"Run id: `{report.run.get('run_id', '')}`", + ]) lines.extend([ "", @@ -196,3 +223,53 @@ def render_markdown(report: OptimizationReport) -> str: "", ]) return "\n".join(lines) + + +def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None: + run_id = str(report.run.get("run_id") or "run") + run_dir = output_path / "runs" / run_id + if run_dir.exists() and report.run.get("mode") == "fake": + shutil.rmtree(run_dir) + run_dir.mkdir(parents=True, exist_ok=True) + + input_paths = report.audit.get("input_paths", {}) + config_path = input_paths.get("optimizer") + if config_path and Path(config_path).is_file(): + shutil.copyfile(config_path, run_dir / "config.snapshot.json") + else: + (run_dir / "config.snapshot.json").write_text("{}", encoding="utf-8") + + (run_dir / "input_hashes.json").write_text( + json.dumps(report.audit.get("input_hashes", {}), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + prompt_dir = run_dir / "candidate_prompts" + results_dir = run_dir / "case_results" + diffs_dir = run_dir / "prompt_diffs" + prompt_dir.mkdir(exist_ok=True) + results_dir.mkdir(exist_ok=True) + diffs_dir.mkdir(exist_ok=True) + + for record in report.candidates: + candidate: CandidatePrompt = record["candidate"] + candidate_dir = prompt_dir / candidate.candidate_id + candidate_dir.mkdir(exist_ok=True) + (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8") + (diffs_dir / f"{candidate.candidate_id}.diff").write_text(candidate.prompt_diff, encoding="utf-8") + for split_name in ("train_result", "validation_result"): + split_result = record[split_name] + path = results_dir / f"{candidate.candidate_id}_{split_result.split}.json" + path.write_text(json.dumps(to_jsonable(split_result), indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float) -> str: + if not baseline_passed and candidate_passed: + return "new_pass" + if baseline_passed and not candidate_passed: + return "new_fail" + if delta > 0: + return "score_up" + if delta < 0: + return "score_down" + return "unchanged" diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py index 31d43cba..b6712233 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -19,6 +19,8 @@ class EvalCase: expectation: dict[str, Any] tags: list[str] = field(default_factory=list) protected: bool = False + simulated_outputs: dict[str, str] = field(default_factory=dict) + expected_failure_category: str | None = None @classmethod def from_dict(cls, payload: dict[str, Any], split: str) -> "EvalCase": @@ -35,6 +37,9 @@ def from_dict(cls, payload: dict[str, Any], split: str) -> "EvalCase": expectation=dict(expectation), tags=list(payload.get("tags") or []), protected=bool(payload.get("protected", False)), + simulated_outputs=dict(payload.get("simulated_outputs") or expectation.get("simulated_outputs") or {}), + expected_failure_category=payload.get("expected_failure_category") + or expectation.get("expected_failure_category"), ) @@ -53,6 +58,7 @@ class CaseResult: evidence: str | None = None cost: float = 0.0 hard_failed: bool = False + expected_failure_category: str | None = None @dataclass(frozen=True) @@ -93,6 +99,7 @@ class CaseDelta: baseline_passed: bool candidate_passed: bool regression: bool + delta_type: str @dataclass(frozen=True) @@ -106,6 +113,12 @@ class GateDecision: validation_score_delta: float new_hard_failures: list[str] protected_regressions: list[str] + validation_new_failures: list[str] + excessive_score_drops: list[str] + overfit_detected: bool + candidate_cost: float + cumulative_cost: float + total_run_cost: float cost: float @@ -115,9 +128,11 @@ class OptimizationReport: schema_version: str run: dict[str, Any] + baseline: dict[str, EvalResult] baseline_train: EvalResult baseline_validation: EvalResult candidates: list[dict[str, Any]] + delta: dict[str, Any] per_case_deltas: list[CaseDelta] failure_attribution_summary: dict[str, Any] gate_decisions: list[GateDecision] diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json index 675e50fc..2d7318d9 100644 --- a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json @@ -1,5 +1,9 @@ { "audit": { + "candidate_prompt_hashes": { + "candidate_001_overfit": "07a494c337544626a4c23afe1a945488e503b06da3e12597419470eb2ab09e42", + "candidate_002_safe": "9669bbd13891b78d2c493df624050dcfd1aa36533987f082a1b78b864df5572f" + }, "candidate_prompts": { "candidate_001_overfit": { "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", @@ -12,7 +16,7 @@ "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." } }, - "config_hash": "1fafb751c9d2b6c63b0daff25844ce64f7f49da32717855ce9b3c50206a6ebe0", + "config_hash": "66a1db3a1c84ad12fd41385fc5e1a9c23cc305e6973945c4bade559c804f9abe", "cost": { "baseline": 0.006, "candidates": { @@ -22,6 +26,18 @@ "total": 0.018 }, "duration_seconds": 0.0, + "input_hashes": { + "optimizer": "a00febaa9efaaf299bbf7dbc08eb31b9093315f0079e601d59c84d0d5704f784", + "prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95", + "train": "03ae8fd41a19ab376c733cb4d5b92a5d6c34431717b5b4899cabdb1e23351a56", + "validation": "73a32b7773f789b2da3d15e925ca0f57da05e8a19c079c9a56ef42055961804b" + }, + "input_paths": { + "optimizer": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\optimizer.json", + "prompt": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", + "train": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\train.evalset.json", + "validation": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\val.evalset.json" + }, "prompt_diffs": { "candidate_001_overfit": "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+ Always force every final answer into JSON, even when the user asks for prose.", "candidate_002_safe": "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+ Use strict JSON only when explicitly requested.\n+ Preserve natural-language answers unless a strict format is requested." @@ -29,12 +45,205 @@ "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", "seed": 91 }, + "baseline": { + "train": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "output": "Here is the JSON you requested: {\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "case_id": "train_json_refund", + "judge": { + "expectation_type": "json", + "valid_json": false + }, + "model": { + "case_id": "train_json_refund", + "expectation_type": "json", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": "expected normalized 'READY', got 'READY - confirmed.'", + "expected_failure_category": "final_response_mismatch", + "failure_category": "final_response_mismatch", + "failure_reason": "normalized exact answer mismatch", + "hard_failed": true, + "output": "READY - confirmed.", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "case_id": "train_exact_order_status", + "judge": { + "expectation_type": "exact", + "expected": "READY" + }, + "model": { + "case_id": "train_exact_order_status", + "expectation_type": "exact", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "latency retries", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_rubric_retry_summary", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "train_rubric_retry_summary", + "expectation_type": "rubric", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.333333, + "split": "train" + }, + "validation": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "output": "Here is the JSON you requested: {\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "case_id": "val_json_invoice", + "judge": { + "expectation_type": "json", + "valid_json": false + }, + "model": { + "case_id": "val_json_invoice", + "expectation_type": "json", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "cache stale data", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_explain_cache", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "val_explain_cache", + "expectation_type": "rubric", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_protected_yes_no", + "judge": { + "expectation_type": "exact", + "expected": "YES" + }, + "model": { + "case_id": "val_protected_yes_no", + "expectation_type": "exact", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.666667, + "split": "validation" + } + }, "baseline_train": { "cases": [ { "case_id": "train_json_refund", "cost": 0.001, "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", "failure_category": "format_violation", "failure_reason": "output is not valid JSON", "hard_failed": true, @@ -63,7 +272,8 @@ "case_id": "train_exact_order_status", "cost": 0.001, "evidence": "expected normalized 'READY', got 'READY - confirmed.'", - "failure_category": "final_answer_mismatch", + "expected_failure_category": "final_response_mismatch", + "failure_category": "final_response_mismatch", "failure_reason": "normalized exact answer mismatch", "hard_failed": true, "output": "READY - confirmed.", @@ -91,10 +301,11 @@ "case_id": "train_rubric_retry_summary", "cost": 0.001, "evidence": null, + "expected_failure_category": null, "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "Latency improved because retries handle transient failures.", + "output": "latency retries", "passed": true, "score": 1.0, "split": "train", @@ -127,6 +338,7 @@ "case_id": "val_json_invoice", "cost": 0.001, "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", "failure_category": "format_violation", "failure_reason": "output is not valid JSON", "hard_failed": true, @@ -155,10 +367,11 @@ "case_id": "val_explain_cache", "cost": 0.001, "evidence": null, + "expected_failure_category": "format_violation", "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "A cache keeps recent data fast, but stale data can appear after updates.", + "output": "cache stale data", "passed": true, "score": 1.0, "split": "validation", @@ -182,6 +395,7 @@ "case_id": "val_protected_yes_no", "cost": 0.001, "evidence": null, + "expected_failure_category": "final_response_mismatch", "failure_category": null, "failure_reason": null, "hard_failed": false, @@ -227,6 +441,7 @@ "case_id": "train_json_refund", "cost": 0.001, "evidence": null, + "expected_failure_category": "format_violation", "failure_category": null, "failure_reason": null, "hard_failed": false, @@ -255,6 +470,7 @@ "case_id": "train_exact_order_status", "cost": 0.001, "evidence": null, + "expected_failure_category": "final_response_mismatch", "failure_category": null, "failure_reason": null, "hard_failed": false, @@ -283,10 +499,11 @@ "case_id": "train_rubric_retry_summary", "cost": 0.001, "evidence": null, + "expected_failure_category": null, "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "Latency improved because retries handle transient failures.", + "output": "latency retries", "passed": true, "score": 1.0, "split": "train", @@ -319,6 +536,7 @@ "case_id": "val_json_invoice", "cost": 0.001, "evidence": null, + "expected_failure_category": "format_violation", "failure_category": null, "failure_reason": null, "hard_failed": false, @@ -347,10 +565,11 @@ "case_id": "val_explain_cache", "cost": 0.001, "evidence": "forbidden pattern '{' was present", + "expected_failure_category": "format_violation", "failure_category": "format_violation", "failure_reason": "output contains a forbidden pattern", "hard_failed": true, - "output": "{\"answer\": \"Cache invalidation prevents stale data.\"}", + "output": "{\"answer\": \"cache stale data\"}", "passed": false, "score": 0.0, "split": "validation", @@ -375,7 +594,8 @@ "case_id": "val_protected_yes_no", "cost": 0.001, "evidence": "expected normalized 'YES', got '{\"answer\": \"YES\"}'", - "failure_category": "final_answer_mismatch", + "expected_failure_category": "final_response_mismatch", + "failure_category": "final_response_mismatch", "failure_reason": "normalized exact answer mismatch", "hard_failed": true, "output": "{\"answer\": \"YES\"}", @@ -420,6 +640,7 @@ "case_id": "train_json_refund", "cost": 0.001, "evidence": null, + "expected_failure_category": "format_violation", "failure_category": null, "failure_reason": null, "hard_failed": false, @@ -448,6 +669,7 @@ "case_id": "train_exact_order_status", "cost": 0.001, "evidence": null, + "expected_failure_category": "final_response_mismatch", "failure_category": null, "failure_reason": null, "hard_failed": false, @@ -476,10 +698,11 @@ "case_id": "train_rubric_retry_summary", "cost": 0.001, "evidence": null, + "expected_failure_category": null, "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "Latency improved because retries handle transient failures.", + "output": "latency retries", "passed": true, "score": 1.0, "split": "train", @@ -512,6 +735,7 @@ "case_id": "val_json_invoice", "cost": 0.001, "evidence": null, + "expected_failure_category": "format_violation", "failure_category": null, "failure_reason": null, "hard_failed": false, @@ -540,10 +764,11 @@ "case_id": "val_explain_cache", "cost": 0.001, "evidence": null, + "expected_failure_category": "format_violation", "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "A cache keeps recent data fast, but stale data can appear after updates.", + "output": "cache stale data", "passed": true, "score": 1.0, "split": "validation", @@ -567,6 +792,7 @@ "case_id": "val_protected_yes_no", "cost": 0.001, "evidence": null, + "expected_failure_category": "final_response_mismatch", "failure_category": null, "failure_reason": null, "hard_failed": false, @@ -600,14 +826,163 @@ } } ], + "delta": { + "per_case": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_explain_cache", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_protected_yes_no", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_explain_cache", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_protected_yes_no", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + } + ] + }, "failure_attribution_summary": { + "attribution_accuracy": 1.0, "by_category": { - "final_answer_mismatch": 2, + "final_response_mismatch": 2, "format_violation": 3 }, "by_prompt_split": { "baseline:train": { - "final_answer_mismatch": 1, + "final_response_mismatch": 1, "format_violation": 1 }, "baseline:validation": { @@ -615,7 +990,7 @@ }, "candidate_001_overfit:train": {}, "candidate_001_overfit:validation": { - "final_answer_mismatch": 1, + "final_response_mismatch": 1, "format_violation": 1 }, "candidate_002_safe:train": {}, @@ -633,7 +1008,7 @@ { "case_id": "train_exact_order_status", "evidence": "expected normalized 'READY', got 'READY - confirmed.'", - "failure_category": "final_answer_mismatch", + "failure_category": "final_response_mismatch", "failure_reason": "normalized exact answer mismatch", "prompt_id": "baseline", "split": "train" @@ -657,46 +1032,65 @@ { "case_id": "val_protected_yes_no", "evidence": "expected normalized 'YES', got '{\"answer\": \"YES\"}'", - "failure_category": "final_answer_mismatch", + "failure_category": "final_response_mismatch", "failure_reason": "normalized exact answer mismatch", "prompt_id": "candidate_001_overfit", "split": "validation" } ], + "expected_labeled_failures": 5, "total_failed_cases": 5 }, "gate_decisions": [ { "accepted": false, + "candidate_cost": 0.006, "candidate_id": "candidate_001_overfit", "cost": 0.006, + "cumulative_cost": 0.006, + "excessive_score_drops": [ + "val_explain_cache", + "val_protected_yes_no" + ], "new_hard_failures": [ "val_explain_cache", "val_protected_yes_no" ], + "overfit_detected": true, "protected_regressions": [ "val_protected_yes_no" ], "reasons": [ - "reject: train score improved but validation score regressed (+0.667 train, -0.333 validation)", + "reject: overfit detected because train score improved but validation score regressed or did not improve (+0.667 train, -0.333 validation)", "reject: validation improvement -0.333 is below required +0.010", "reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no']", "reject: protected cases regressed: ['val_protected_yes_no']", "reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no']" ], + "total_run_cost": 0.012, "train_score_delta": 0.666667, + "validation_new_failures": [ + "val_explain_cache", + "val_protected_yes_no" + ], "validation_score_delta": -0.333334 }, { "accepted": true, + "candidate_cost": 0.006, "candidate_id": "candidate_002_safe", "cost": 0.006, + "cumulative_cost": 0.012, + "excessive_score_drops": [], "new_hard_failures": [], + "overfit_detected": false, "protected_regressions": [], "reasons": [ "accept: validation score improved +0.333 with no protected regression or new hard failure" ], + "total_run_cost": 0.018, "train_score_delta": 0.666667, + "validation_new_failures": [], "validation_score_delta": 0.333333 } ], @@ -709,6 +1103,7 @@ "candidate_score": 1.0, "case_id": "train_json_refund", "delta": 1.0, + "delta_type": "new_pass", "regression": false, "split": "train" }, @@ -720,6 +1115,7 @@ "candidate_score": 1.0, "case_id": "train_exact_order_status", "delta": 1.0, + "delta_type": "new_pass", "regression": false, "split": "train" }, @@ -731,6 +1127,7 @@ "candidate_score": 1.0, "case_id": "train_rubric_retry_summary", "delta": 0.0, + "delta_type": "unchanged", "regression": false, "split": "train" }, @@ -742,6 +1139,7 @@ "candidate_score": 1.0, "case_id": "val_json_invoice", "delta": 1.0, + "delta_type": "new_pass", "regression": false, "split": "validation" }, @@ -753,6 +1151,7 @@ "candidate_score": 0.0, "case_id": "val_explain_cache", "delta": -1.0, + "delta_type": "new_fail", "regression": true, "split": "validation" }, @@ -764,6 +1163,7 @@ "candidate_score": 0.0, "case_id": "val_protected_yes_no", "delta": -1.0, + "delta_type": "new_fail", "regression": true, "split": "validation" }, @@ -775,6 +1175,7 @@ "candidate_score": 1.0, "case_id": "train_json_refund", "delta": 1.0, + "delta_type": "new_pass", "regression": false, "split": "train" }, @@ -786,6 +1187,7 @@ "candidate_score": 1.0, "case_id": "train_exact_order_status", "delta": 1.0, + "delta_type": "new_pass", "regression": false, "split": "train" }, @@ -797,6 +1199,7 @@ "candidate_score": 1.0, "case_id": "train_rubric_retry_summary", "delta": 0.0, + "delta_type": "unchanged", "regression": false, "split": "train" }, @@ -808,6 +1211,7 @@ "candidate_score": 1.0, "case_id": "val_json_invoice", "delta": 1.0, + "delta_type": "new_pass", "regression": false, "split": "validation" }, @@ -819,6 +1223,7 @@ "candidate_score": 1.0, "case_id": "val_explain_cache", "delta": 0.0, + "delta_type": "unchanged", "regression": false, "split": "validation" }, @@ -830,6 +1235,7 @@ "candidate_score": 1.0, "case_id": "val_protected_yes_no", "delta": 0.0, + "delta_type": "unchanged", "regression": false, "split": "validation" } @@ -838,10 +1244,17 @@ "fake_judge": true, "fake_model": true, "mode": "fake", - "prompt_source": "prompts/baseline_system_prompt.txt", + "paths": { + "optimizer": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\optimizer.json", + "prompt": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", + "train": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\train.evalset.json", + "validation": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\val.evalset.json" + }, + "prompt_source": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", "run_id": "eval_optimize_loop_seed_91", "trace_enabled": true, "train_cases": 3, + "update_source": false, "validation_cases": 3 }, "schema_version": "eval_optimize_loop.v1", diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md index 7c5fc8cf..a4e8e862 100644 --- a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md @@ -4,10 +4,13 @@ Selected candidate: `candidate_002_safe`. +Update source prompt: no (default) + + ## Gate Reasons ### candidate_001_overfit (rejected) -- reject: train score improved but validation score regressed (+0.667 train, -0.333 validation) +- reject: overfit detected because train score improved but validation score regressed or did not improve (+0.667 train, -0.333 validation) - reject: validation improvement -0.333 is below required +0.010 - reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no'] - reject: protected cases regressed: ['val_protected_yes_no'] @@ -26,20 +29,20 @@ Selected candidate: `candidate_002_safe`. ## Per-Case Delta -| candidate | split | case | baseline | candidate | delta | passed -> passed | -| --- | --- | --- | ---: | ---: | ---: | --- | -| candidate_001_overfit | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | -| candidate_001_overfit | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | -| candidate_001_overfit | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | -| candidate_001_overfit | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | -| candidate_001_overfit | validation | val_explain_cache | 1.000 | 0.000 | -1.000 | True -> False | -| candidate_001_overfit | validation | val_protected_yes_no | 1.000 | 0.000 | -1.000 | True -> False | -| candidate_002_safe | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | -| candidate_002_safe | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | -| candidate_002_safe | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | -| candidate_002_safe | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | -| candidate_002_safe | validation | val_explain_cache | 1.000 | 1.000 | +0.000 | True -> True | -| candidate_002_safe | validation | val_protected_yes_no | 1.000 | 1.000 | +0.000 | True -> True | +| candidate | split | case | baseline | candidate | delta | passed -> passed | delta type | +| --- | --- | --- | ---: | ---: | ---: | --- | --- | +| candidate_001_overfit | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_001_overfit | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | validation | val_explain_cache | 1.000 | 0.000 | -1.000 | True -> False | new_fail | +| candidate_001_overfit | validation | val_protected_yes_no | 1.000 | 0.000 | -1.000 | True -> False | new_fail | +| candidate_002_safe | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_002_safe | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | validation | val_explain_cache | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_002_safe | validation | val_protected_yes_no | 1.000 | 1.000 | +0.000 | True -> True | unchanged | ## Failure Attribution Summary @@ -47,9 +50,17 @@ Total failed case evaluations: 5 | category | count | | --- | ---: | -| final_answer_mismatch | 2 | +| final_response_mismatch | 2 | | format_violation | 3 | +Attribution accuracy: 1.000 + +## Cost And Audit + +Total cost: 0.018 +Config hash: `66a1db3a1c84ad12fd41385fc5e1a9c23cc305e6973945c4bade559c804f9abe` +Run id: `eval_optimize_loop_seed_91` + ## Prompt Diff ### candidate_001_overfit diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index e8aaa3ae..17e704be 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import hashlib import sys import tempfile from pathlib import Path @@ -12,15 +13,15 @@ if str(HERE) not in sys.path: sys.path.insert(0, str(HERE)) -from eval_loop.evaluator import ExampleEvaluator -from eval_loop.fake_judge import FakeJudge -from eval_loop.fake_model import FakeModel +from eval_loop.backends import FakeBackend +from eval_loop.backends import SDKBackend +from eval_loop.config import validate_inputs from eval_loop.gate import AcceptanceGate from eval_loop.loader import load_eval_cases from eval_loop.loader import load_optimizer_config from eval_loop.loader import load_prompt +from eval_loop.loader import sha256_file from eval_loop.loader import stable_config_hash -from eval_loop.optimizer import FakeOptimizer from eval_loop.report import REPRODUCIBILITY_COMMAND from eval_loop.report import build_report from eval_loop.report import compute_case_deltas @@ -43,27 +44,55 @@ def run_pipeline( optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, prompt_path: str | Path = DEFAULT_PROMPT, output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", fake_model: bool = True, fake_judge: bool = True, trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, ) -> OptimizationReport: """Run baseline eval, fake optimization, validation gate, and reports.""" - if not fake_model or not fake_judge: + if mode not in {"fake", "sdk"}: + raise ValueError("field 'mode' must be one of: fake, sdk") + if mode == "fake" and (not fake_model or not fake_judge): raise ValueError( - "This example-local implementation only supports offline fake mode. " - "Pass --fake-model --fake-judge, or replace ExampleEvaluator with a real adapter." + "fake mode requires fake_model=True and fake_judge=True. Pass --fake-model --fake-judge " + "or use --mode sdk with --sdk-call-agent module:function." ) train_cases = load_eval_cases(train_path, split="train") validation_cases = load_eval_cases(val_path, split="validation") - if len(train_cases) < 3 or len(validation_cases) < 3: - raise ValueError("example requires at least 3 train and 3 validation eval cases") - optimizer_config = load_optimizer_config(optimizer_config_path) - seed = int(optimizer_config.get("seed", 91)) + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + train_cases=train_cases, + validation_cases=validation_cases, + config=optimizer_config, + ) + if mode == "sdk": + SDKBackend( + prompt_path=prompt_path, + call_agent_path=sdk_call_agent, + update_source=update_source, + ).optimize( + baseline_prompt=load_prompt(prompt_path), + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + output_dir=output_dir, + ) + raise ValueError( + "sdk mode delegated optimization to AgentOptimizer. Inspect the SDK artifacts in output_dir; " + "the example JSON/Markdown report is generated by fake mode. To run offline review artifacts, " + "use --mode fake --trace." + ) + + seed = optimizer_config.seed baseline_prompt = load_prompt(prompt_path) - evaluator = ExampleEvaluator(FakeModel(seed=seed), FakeJudge(), trace_enabled=trace) + backend = FakeBackend(seed=seed, trace_enabled=trace) baseline = CandidatePrompt( candidate_id="baseline", @@ -71,34 +100,40 @@ def run_pipeline( rationale="Prompt source file before optimization.", prompt_diff="", ) - baseline_train = evaluator.evaluate( + baseline_train = backend.evaluate( prompt_id=baseline.candidate_id, prompt=baseline.prompt, cases=train_cases, split="train", ) - baseline_validation = evaluator.evaluate( + baseline_validation = backend.evaluate( prompt_id=baseline.candidate_id, prompt=baseline.prompt, cases=validation_cases, split="validation", ) - optimizer = FakeOptimizer() - candidates = optimizer.propose(baseline_prompt) - gate = AcceptanceGate(optimizer_config["gate"]) + candidates = backend.optimize( + baseline_prompt=baseline_prompt, + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + output_dir=output_dir, + ) + gate = AcceptanceGate(optimizer_config.gate.to_dict()) candidate_records: list[dict[str, Any]] = [] all_deltas = [] gate_decisions = [] + cumulative_cost = round(baseline_train.cost + baseline_validation.cost, 6) for candidate in candidates: - train_result = evaluator.evaluate( + train_result = backend.evaluate( prompt_id=candidate.candidate_id, prompt=candidate.prompt, cases=train_cases, split="train", ) - validation_result = evaluator.evaluate( + validation_result = backend.evaluate( prompt_id=candidate.candidate_id, prompt=candidate.prompt, cases=validation_cases, @@ -118,6 +153,7 @@ def run_pipeline( candidate_train=train_result, candidate_validation=validation_result, deltas=deltas, + cumulative_cost=cumulative_cost, ) candidate_records.append({ "candidate": candidate, @@ -126,25 +162,46 @@ def run_pipeline( }) all_deltas.extend(deltas) gate_decisions.append(decision) + cumulative_cost = decision.total_run_cost selected_candidate = _select_candidate(candidate_records, gate_decisions) + input_hashes = _input_hashes( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + ) audit = _build_audit( seed=seed, - config_hash=stable_config_hash(optimizer_config), + config_hash=stable_config_hash(optimizer_config.to_dict()), baseline_train=baseline_train, baseline_validation=baseline_validation, candidate_records=candidate_records, candidates=candidates, + input_hashes=input_hashes, + input_paths={ + "train": str(train_path), + "validation": str(val_path), + "optimizer": str(optimizer_config_path), + "prompt": str(prompt_path), + }, ) run = { "run_id": f"eval_optimize_loop_seed_{seed}", - "mode": "fake", + "mode": mode, "fake_model": fake_model, "fake_judge": fake_judge, "trace_enabled": trace, "train_cases": len(train_cases), "validation_cases": len(validation_cases), - "prompt_source": "prompts/baseline_system_prompt.txt", + "update_source": update_source, + "paths": { + "train": str(train_path), + "validation": str(val_path), + "optimizer": str(optimizer_config_path), + "prompt": str(prompt_path), + }, + "prompt_source": str(prompt_path), } report = build_report( run=run, @@ -189,6 +246,8 @@ def _build_audit( baseline_validation, candidate_records: list[dict[str, Any]], candidates: list[CandidatePrompt], + input_hashes: dict[str, str], + input_paths: dict[str, str], ) -> dict[str, Any]: baseline_cost = round(baseline_train.cost + baseline_validation.cost, 6) candidate_costs = { @@ -196,10 +255,17 @@ def _build_audit( for record in candidate_records } total_cost = round(baseline_cost + sum(candidate_costs.values()), 6) + candidate_prompt_hashes = { + candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() + for candidate in candidates + } return { "seed": seed, "duration_seconds": 0.0, "config_hash": config_hash, + "input_hashes": input_hashes, + "input_paths": input_paths, + "candidate_prompt_hashes": candidate_prompt_hashes, "cost": { "baseline": baseline_cost, "candidates": candidate_costs, @@ -228,12 +294,30 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--optimizer-config", default=str(DEFAULT_OPTIMIZER_CONFIG), help="Path to optimizer.json") parser.add_argument("--prompt", default=str(DEFAULT_PROMPT), help="Path to baseline system prompt") parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR), help="Directory for runtime reports") + parser.add_argument("--mode", choices=("fake", "sdk"), default="fake", help="Backend mode") parser.add_argument("--fake-model", action="store_true", help="Use deterministic fake model") parser.add_argument("--fake-judge", action="store_true", help="Use deterministic fake judge") parser.add_argument("--trace", action="store_true", help="Persist fake model/judge trace details per case") + parser.add_argument("--sdk-call-agent", help="Async call_agent target for SDK mode, as module:function") + parser.add_argument("--update-source", action="store_true", help="Allow SDK optimizer to write back source prompt") return parser.parse_args(argv) +def _input_hashes( + *, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + prompt_path: str | Path, +) -> dict[str, str]: + return { + "train": sha256_file(train_path), + "validation": sha256_file(val_path), + "optimizer": sha256_file(optimizer_config_path), + "prompt": sha256_file(prompt_path), + } + + def main(argv: list[str] | None = None) -> OptimizationReport: args = parse_args(argv) report = run_pipeline( @@ -242,9 +326,12 @@ def main(argv: list[str] | None = None) -> OptimizationReport: optimizer_config_path=args.optimizer_config, prompt_path=args.prompt, output_dir=args.output_dir, - fake_model=args.fake_model, - fake_judge=args.fake_judge, + mode=args.mode, + fake_model=args.fake_model or args.mode == "fake", + fake_judge=args.fake_judge or args.mode == "fake", trace=args.trace, + sdk_call_agent=args.sdk_call_agent, + update_source=args.update_source, ) output_dir = Path(args.output_dir) print(f"Wrote {output_dir / 'optimization_report.json'}") diff --git a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py new file mode 100644 index 00000000..1ce4825f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.config import parse_optimizer_config +from examples.optimization.eval_optimize_loop.eval_loop.config import validate_inputs +from examples.optimization.eval_optimize_loop.eval_loop.loader import load_eval_cases +from examples.optimization.eval_optimize_loop.eval_loop.loader import load_optimizer_config + + +def test_optimizer_config_defaults_metrics_and_gate(tmp_path: Path): + path = tmp_path / "optimizer.json" + path.write_text(json.dumps({"seed": 7}), encoding="utf-8") + + config = load_optimizer_config(path) + + assert config.seed == 7 + assert config.metrics == {} + assert config.gate.min_val_score_improvement == 0.01 + + +def test_optimizer_config_rejects_bad_gate_type(tmp_path: Path): + path = tmp_path / "optimizer.json" + payload = {"gate": {"allow_new_hard_fail": "no"}} + + with pytest.raises(ValueError, match="gate.allow_new_hard_fail"): + parse_optimizer_config(payload, path=path) + + +def test_optimizer_config_rejects_negative_cost(tmp_path: Path): + path = tmp_path / "optimizer.json" + payload = {"gate": {"max_total_cost": -1}} + + with pytest.raises(ValueError, match="gate.max_total_cost"): + parse_optimizer_config(payload, path=path) + + +def test_validate_inputs_rejects_same_train_val_path(tmp_path: Path): + eval_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) + config = parse_optimizer_config({"gate": {}}, path=tmp_path / "optimizer.json") + cases = load_eval_cases(eval_path, split="train") + + with pytest.raises(ValueError, match="must be different"): + validate_inputs( + train_path=eval_path, + val_path=eval_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=cases, + validation_cases=cases, + config=config, + ) + + +def test_validate_inputs_rejects_duplicate_case_ids(tmp_path: Path): + train_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "a", "c"]) + val_path = _write_evalset(tmp_path / "val.evalset.json", "validation", ["v1", "v2", "v3"]) + config = parse_optimizer_config({"gate": {}}, path=tmp_path / "optimizer.json") + + with pytest.raises(ValueError, match="duplicate case_id"): + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=load_eval_cases(train_path, split="train"), + validation_cases=load_eval_cases(val_path, split="validation"), + config=config, + ) + + +def test_validate_inputs_rejects_missing_protected_case(tmp_path: Path): + train_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) + val_path = _write_evalset(tmp_path / "val.evalset.json", "validation", ["v1", "v2", "v3"]) + config = parse_optimizer_config( + {"gate": {"protected_case_ids": ["missing"]}}, + path=tmp_path / "optimizer.json", + ) + + with pytest.raises(ValueError, match="gate.protected_case_ids"): + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=load_eval_cases(train_path, split="train"), + validation_cases=load_eval_cases(val_path, split="validation"), + config=config, + ) + + +def _write_evalset(path: Path, split: str, ids: list[str]) -> Path: + payload = { + "split": split, + "cases": [ + { + "id": case_id, + "input": "Return JSON", + "expectation": {"type": "json", "required_keys": ["answer"], "expected_values": {"answer": case_id}}, + } + for case_id in ids + ], + } + path.write_text(json.dumps(payload), encoding="utf-8") + return path diff --git a/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py index 22356be8..60cbeaf5 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py +++ b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py @@ -1,6 +1,7 @@ from __future__ import annotations from examples.optimization.eval_optimize_loop.eval_loop.attribution import attribute_failure +from examples.optimization.eval_optimize_loop.eval_loop.attribution import summarize_failures from examples.optimization.eval_optimize_loop.eval_loop.evaluator import ExampleEvaluator from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel @@ -9,7 +10,7 @@ def test_failure_attribution_labels_format_violation_and_exact_mismatch(): assert attribute_failure("json_parse_failure", "bad json")[0] == "format_violation" - assert attribute_failure("exact_answer_mismatch", "wrong answer")[0] == "final_answer_mismatch" + assert attribute_failure("exact_answer_mismatch", "wrong answer")[0] == "final_response_mismatch" def test_evaluator_attaches_failure_details_to_failed_cases(): @@ -42,6 +43,66 @@ def test_evaluator_attaches_failure_details_to_failed_cases(): assert by_id["json_case"].failure_category == "format_violation" assert by_id["json_case"].failure_reason assert by_id["json_case"].evidence - assert by_id["exact_case"].failure_category == "final_answer_mismatch" + assert by_id["exact_case"].failure_category == "final_response_mismatch" assert by_id["exact_case"].failure_reason assert by_id["exact_case"].evidence + + +def test_failure_attribution_labels_tool_parameter_and_knowledge_failures(): + assert attribute_failure("tool_call_error", "wrong tool")[0] == "tool_call_error" + assert attribute_failure("parameter_error", "wrong arg")[0] == "parameter_error" + assert attribute_failure("knowledge_recall_insufficient", "missing source")[0] == "knowledge_recall_insufficient" + + +def test_fake_judge_scores_tool_and_parameter_expectations(): + judge = FakeJudge() + tool_case = EvalCase( + case_id="tool_case", + split="train", + input="Call tool", + expectation={"type": "tool", "expected_tool": "lookup", "expected_args": {"id": "42"}}, + ) + + assert judge.score(tool_case, '{"tool": "lookup", "args": {"id": "42"}}').passed + wrong_tool = judge.score(tool_case, '{"tool": "search", "args": {"id": "42"}}') + wrong_arg = judge.score(tool_case, '{"tool": "lookup", "args": {"id": "43"}}') + assert wrong_tool.error_code == "tool_call_error" + assert wrong_arg.error_code == "parameter_error" + + +def test_fake_judge_scores_knowledge_expectations(): + case = EvalCase( + case_id="knowledge_case", + split="train", + input="Recall source", + expectation={ + "type": "knowledge", + "required_sources": ["doc-a"], + "must_include_knowledge_terms": ["refund"], + }, + ) + + assert FakeJudge().score(case, "refund policy appears in doc-a").passed + failed = FakeJudge().score(case, "refund policy only") + assert failed.error_code == "knowledge_recall_insufficient" + + +def test_failure_summary_computes_attribution_accuracy(): + result = ExampleEvaluator(FakeModel(seed=91), FakeJudge()).evaluate( + prompt_id="baseline", + prompt="baseline prompt", + split="train", + cases=[ + EvalCase( + case_id="json_labeled", + split="train", + input="Return JSON", + expectation={"type": "json", "expected_values": {"answer": "ok"}}, + expected_failure_category="format_violation", + ) + ], + ) + + summary = summarize_failures([result]) + assert summary["expected_labeled_failures"] == 1 + assert summary["attribution_accuracy"] == 1.0 diff --git a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py new file mode 100644 index 00000000..5bd26f9a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json + +from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase + + +def test_fake_model_json_behavior_does_not_depend_on_case_id(): + case = EvalCase( + case_id="hidden_json_case", + split="validation", + input="Return JSON.", + expectation={"type": "json", "expected_values": {"status": "ok"}, "required_keys": ["status"]}, + ) + model = FakeModel(seed=91) + + baseline, _, _ = model.generate("baseline", "plain prompt", case) + safe, _, _ = model.generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) + + assert baseline.startswith("Here is the JSON") + assert json.loads(safe) == {"status": "ok"} + assert FakeJudge().score(case, baseline).passed is False + assert FakeJudge().score(case, safe).passed is True + + +def test_fake_model_protected_exact_baseline_passes_but_overfit_regresses(): + case = EvalCase( + case_id="hidden_protected_case", + split="validation", + input="Answer exactly YES.", + expectation={"type": "exact", "expected": "YES"}, + protected=True, + ) + model = FakeModel(seed=91) + + baseline, _, _ = model.generate("baseline", "plain prompt", case) + overfit, _, _ = model.generate("overfit", "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON", case) + safe, _, _ = model.generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) + + assert baseline == "YES" + assert overfit == '{"answer": "YES"}' + assert safe == "YES" + + +def test_fake_model_rubric_overfit_forces_json_on_validation_prose(): + case = EvalCase( + case_id="hidden_rubric_case", + split="validation", + input="Explain in prose.", + expectation={ + "type": "rubric", + "must_include": ["cache", "stale data"], + "forbidden": ["{", "}", "json"], + "max_chars": 120, + }, + tags=["prose"], + ) + + output, _, _ = FakeModel(seed=91).generate("overfit", "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON", case) + assert output.startswith("{") + judged = FakeJudge().score(case, output) + assert judged.error_code == "forbidden_pattern" + + +def test_fake_model_respects_simulated_outputs_override(): + case = EvalCase( + case_id="override_case", + split="train", + input="Any", + expectation={"type": "exact", "expected": "OK"}, + simulated_outputs={"safe": "CUSTOM"}, + ) + + output, _, _ = FakeModel(seed=91).generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) + assert output == "CUSTOM" diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py index 041d4ec1..40b5adfb 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_gate.py +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -30,6 +30,7 @@ def test_gate_rejects_train_improvement_with_validation_regression(): assert not decision.accepted assert any("train score improved but validation score regressed" in reason for reason in decision.reasons) + assert decision.overfit_detected def test_gate_rejects_protected_case_regression(): @@ -86,6 +87,108 @@ def test_gate_accepts_safe_candidate(): assert decision.new_hard_failures == [] +def test_gate_rejects_new_hard_failure_when_not_allowed(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 0.0), ("val_c", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 0.0), ("val_b", 1.0), ("val_c", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({"allow_new_hard_fail": False}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert not decision.accepted + assert decision.new_hard_failures == ["val_a"] + assert decision.validation_new_failures == ["val_a"] + + +def test_gate_rejects_excessive_score_drop(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 0.0), ("val_c", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 0.4), ("val_b", 1.0), ("val_c", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({"max_score_drop_per_case": 0.5, "allow_new_hard_fail": True}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert not decision.accepted + assert decision.excessive_score_drops == ["val_a"] + + +def test_gate_rejects_cost_budget(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({"max_total_cost": 0.001}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cumulative_cost=0.001, + ) + + assert not decision.accepted + assert decision.total_run_cost > 0.001 + + +def test_case_delta_types_are_classified(): + baseline_train = _eval("baseline", "train", [("new_pass", 0.0), ("new_fail", 1.0)]) + candidate_train = _eval("candidate", "train", [("new_pass", 1.0), ("new_fail", 0.0)]) + baseline_val = _eval("baseline", "validation", [("up", 0.2), ("down", 0.8), ("same", 0.5)]) + candidate_val = _eval("candidate", "validation", [("up", 0.5), ("down", 0.5), ("same", 0.5)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + by_case = {delta.case_id: delta.delta_type for delta in deltas} + assert by_case["new_pass"] == "new_pass" + assert by_case["new_fail"] == "new_fail" + assert by_case["up"] == "score_up" + assert by_case["down"] == "score_down" + assert by_case["same"] == "unchanged" + + def _eval(prompt_id: str, split: str, scores: list[tuple[str, float]]) -> EvalResult: cases = [ CaseResult( diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py index 62bed2a0..6a6ce68c 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -2,6 +2,8 @@ import json from pathlib import Path +import subprocess +import sys from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT @@ -35,8 +37,10 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): "run", "baseline_train", "baseline_validation", + "baseline", "candidates", "per_case_deltas", + "delta", "failure_attribution_summary", "gate_decisions", "selected_candidate", @@ -51,6 +55,7 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): ] assert len(payload["per_case_deltas"]) == 12 assert payload["failure_attribution_summary"]["by_category"]["format_violation"] >= 1 + assert payload["failure_attribution_summary"]["attribution_accuracy"] == 1.0 assert set(payload["audit"]) >= { "seed", "config_hash", @@ -58,6 +63,8 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): "duration_seconds", "candidate_prompts", "prompt_diffs", + "input_hashes", + "candidate_prompt_hashes", } assert "candidate_001_overfit" in payload["audit"]["prompt_diffs"] assert "candidate_002_safe" in payload["audit"]["prompt_diffs"] @@ -79,6 +86,14 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert "Failure Attribution Summary" in markdown assert "Prompt Diff" in markdown assert "Reproducibility" in markdown + assert "Cost And Audit" in markdown + + run_dir = output_dir / "runs" / "eval_optimize_loop_seed_91" + assert (run_dir / "config.snapshot.json").is_file() + assert (run_dir / "input_hashes.json").is_file() + assert (run_dir / "candidate_prompts" / "candidate_001_overfit" / "system_prompt.txt").is_file() + assert (run_dir / "case_results" / "candidate_002_safe_validation.json").is_file() + assert (run_dir / "prompt_diffs" / "candidate_002_safe.diff").is_file() def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): @@ -93,3 +108,43 @@ def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): assert (first_dir / "optimization_report.md").read_text(encoding="utf-8") == ( second_dir / "optimization_report.md" ).read_text(encoding="utf-8") + + +def test_pipeline_accepts_mode_fake_without_legacy_flags(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + assert report.run["mode"] == "fake" + assert report.selected_candidate == "candidate_002_safe" + + +def test_pipeline_selected_candidate_is_null_when_all_candidates_rejected(tmp_path: Path): + config_path = tmp_path / "optimizer.json" + config = json.loads(Path(DEFAULT_OPTIMIZER_CONFIG).read_text(encoding="utf-8")) + config["gate"]["min_val_score_improvement"] = 1.0 + config_path.write_text(json.dumps(config), encoding="utf-8") + + report = run_pipeline( + optimizer_config_path=config_path, + output_dir=tmp_path / "run", + mode="fake", + trace=True, + ) + + assert report.selected_candidate is None + + +def test_cli_mode_fake_and_legacy_fake_flags_both_run(tmp_path: Path): + script = Path("examples/optimization/eval_optimize_loop/run_pipeline.py") + first = tmp_path / "first" + second = tmp_path / "second" + + subprocess.run( + [sys.executable, str(script), "--mode", "fake", "--trace", "--output-dir", str(first)], + check=True, + ) + subprocess.run( + [sys.executable, str(script), "--fake-model", "--fake-judge", "--trace", "--output-dir", str(second)], + check=True, + ) + + assert (first / "optimization_report.json").is_file() + assert (second / "optimization_report.md").is_file() diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py new file mode 100644 index 00000000..a4be6eab --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import sys +import types +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.backends import SDKBackend + + +def test_sdk_backend_requires_call_agent_path(tmp_path: Path): + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt") + + with pytest.raises(ValueError, match="--sdk-call-agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_calls_agent_optimizer_and_converts_best_prompt(tmp_path: Path, monkeypatch): + calls = {} + + class FakeTargetPrompt: + def __init__(self): + self.paths = [] + + def add_path(self, name, path): + self.paths.append((name, path)) + return self + + class FakeAgentOptimizer: + @staticmethod + async def optimize(**kwargs): + calls.update(kwargs) + return types.SimpleNamespace(best_prompts={"system_prompt": "optimized prompt"}) + + fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") + fake_eval_module.AgentEvaluator = object + fake_eval_module.AgentOptimizer = FakeAgentOptimizer + fake_eval_module.TargetPrompt = FakeTargetPrompt + monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) + + call_agent_module = types.ModuleType("fake_call_agent_module") + + async def call_agent(query: str) -> str: + return query + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + candidates = backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["config_path"].endswith("optimizer.json") + assert calls["update_source"] is False + assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] + assert candidates[0].candidate_id == "sdk_best" + assert candidates[0].prompt == "optimized prompt" From 843de994e8c0d65759e5bd7310f1133e9877bb95 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 21:10:22 +0800 Subject: [PATCH 04/34] Polish eval optimize loop SDK path --- .gitignore | 1 + .../optimization/eval_optimize_loop/DESIGN.md | 15 +- .../optimization/eval_optimize_loop/README.md | 15 +- .../eval_optimize_loop/eval_loop/backends.py | 123 ++++++--- .../eval_optimize_loop/eval_loop/diffing.py | 19 ++ .../eval_optimize_loop/eval_loop/optimizer.py | 52 ++-- .../eval_optimize_loop/eval_loop/report.py | 12 +- .../eval_optimize_loop/eval_loop/schemas.py | 2 + .../outputs/optimization_report.example.json | 21 +- .../outputs/optimization_report.example.md | 27 +- .../eval_optimize_loop/run_pipeline.py | 242 +++++++++++++++-- .../eval_optimize_loop/tests/test_gate.py | 1 + .../test_no_sample_case_id_hardcoding.py | 133 +++++++++ .../tests/test_pipeline_fake_mode.py | 9 + .../tests/test_sdk_backend.py | 253 ++++++++++++++++-- 15 files changed, 802 insertions(+), 123 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/diffing.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py diff --git a/.gitignore b/.gitignore index eb3054c4..1e9b3f83 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ examples/*.log examples/optimization/eval_optimize_loop/outputs/optimization_report.json examples/optimization/eval_optimize_loop/outputs/optimization_report.md +examples/optimization/eval_optimize_loop/outputs/runs/ trpc-agent-py.egg-info diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 32df0c80..42a0e80a 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -1,3 +1,16 @@ # 设计说明 -本示例把闭环拆成加载、评测、归因、候选生成、门禁和报告六层。失败归因采用可审计规则表：JSON 解析和禁用格式命中归为 `format_violation`，最终答案错误归为 `final_response_mismatch`，工具名错误、参数错误、知识召回不足和超长分别进入独立类别；若用例声明期望归因，报告会计算 accuracy。接受门禁先要求 validation 提升，再拒绝 train 提升但 validation 不提升的过拟合候选，并检查受保护用例降分、新硬失败、单例最大降分和成本预算。防过拟合依赖训练集与验证集分离、protected case、delta_type 和逐例降分阈值，而不是只看平均分；每个拒绝理由都会进入审计报告，便于复盘，也便于评审核对策略。fake mode 用通用 expectation/tags/protected/simulated_outputs 生成输出，不依赖样例名，可稳定覆盖隐藏样本；sdk mode 则通过 `SDKBackend` 调用 `AgentOptimizer` 与 `TargetPrompt`，用于真实接入，两者共享门禁、归因和报告结构。报告写 JSON、Markdown 和 runs 审计目录，保存输入哈希、配置快照、候选 prompt、diff、case 结果和成本。默认不回写源 prompt，只有显式 `--update-source` 才允许，避免示例运行污染源码。 +## 失败归因 +评测先由 FakeJudge 产出结构化失败，再用规则归因：JSON、禁用格式归为 `format_violation`，精确答案错归为 `final_response_mismatch`，工具名、参数、知识召回、长度和 rubric 各有独立类别。每个失败都保留 reason 与 evidence，便于复核。若样例声明期望类别，报告会计算归因准确率。 + +## 接受门禁 +Gate 要求验证集提升达到阈值，并检查新硬失败、受保护样例退化、单例降分和累计成本。若训练分上涨但验证不涨，直接标记过拟合并拒绝。 + +## 防过拟合 +训练集只用于暴露问题，候选必须通过验证集和 protected case。过拟合候选即使修好训练格式，只要把验证集自然语言或受保护精确答案改坏，也会被拒绝。`delta_type` 标出 new_pass、new_fail、score_up、score_down，避免只看平均分掩盖局部退化。 + +## Fake 与 SDK +Fake mode 由 expectation、tags、protected 和 simulated_outputs 驱动，不依赖样例 id，可无 API key 稳定复现。SDK mode 通过 `SDKBackend` 调用 `AgentOptimizer` 与 `TargetPrompt`，失败时给出明确错误，不回退到 fake。 + +## 审计与回写 +报告同时写 JSON、Markdown 和 `runs//`，保存输入哈希、配置快照、候选 prompt、diff、case 结果与成本，并给出可复现命令。默认不回写源 prompt，只有显式 `--update-source` 才允许，报告会记录该选择。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index ad4f1bde..f0fdc43a 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -68,7 +68,11 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ `--sdk-call-agent` must point to an async callable compatible with `AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials -needed by that callable. SDK mode never silently falls back to fake mode. +needed by that callable. SDK mode never silently falls back to fake mode. When +the SDK optimizer returns a best prompt but does not expose per-case validation +results, this example still writes JSON/Markdown reports and audit artifacts +with `gate_status: not_applicable`, records the SDK result summary, and selects +`sdk_best` as the SDK optimizer's chosen prompt. ## Source Prompt Writes @@ -110,6 +114,11 @@ optional `simulated_outputs`; it does not depend on sample `case_id` names. - audit data: seed, duration, config hash, input hashes, candidate prompt hashes, cost, prompt diffs, and reproducibility command. +`gate.max_total_cost` is interpreted as the total evaluated run cost at the time +each candidate is judged: baseline cost plus all candidates evaluated so far, +including rejected candidates. This makes budget decisions deterministic and +auditable when multiple candidates are considered. + `optimization_report.md` includes final decision, gate reasons, score table, per-case delta table, failure attribution summary, cost/audit details, prompt diffs, and the reproducibility command. @@ -139,4 +148,6 @@ python -m pytest examples/optimization/eval_optimize_loop/tests The tests cover fake hidden-sample generalization, config validation, gate rejection paths, protected-case behavior, failure attribution, tool/knowledge judge paths, SDK adapter wiring through monkeypatching, deterministic report -generation, and both CLI forms. +generation, and both CLI forms. CI can run fake mode plus the monkeypatched SDK +smoke tests without real API credentials; real SDK/model calls are opt-in local +or integration runs. diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 38238c05..1e11ff41 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -8,8 +8,8 @@ from pathlib import Path from typing import Any from typing import Iterable -from typing import Protocol +from .diffing import make_unified_diff from .evaluator import ExampleEvaluator from .fake_judge import FakeJudge from .fake_model import FakeModel @@ -19,22 +19,6 @@ from .schemas import EvalResult -class EvalOptimizeBackend(Protocol): - def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: - ... - - def optimize( - self, - *, - baseline_prompt: str, - train_path: str | Path, - val_path: str | Path, - optimizer_config_path: str | Path, - output_dir: str | Path, - ) -> list[CandidatePrompt]: - ... - - @dataclass class FakeBackend: seed: int = 91 @@ -61,20 +45,43 @@ def optimize( @dataclass class SDKBackend: - """Thin adapter around AgentOptimizer/TargetPrompt for real SDK runs.""" + """Thin optimizer adapter around AgentOptimizer/TargetPrompt for SDK runs. + + SDK mode relies on AgentOptimizer's internal evaluation loop. It does not + implement the fake per-case ``evaluate`` API. + """ prompt_path: str | Path call_agent_path: str | None = None update_source: bool = False + last_result_summary: dict[str, Any] | None = None + last_artifact_dir: str | None = None - def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: - raise ValueError( - "sdk mode evaluation expects SDK evalset files and is performed by AgentOptimizer/AgentEvaluator. " - "Use --sdk-call-agent module:function and SDK-compatible train/val evalsets; fake EvalCase objects " - "cannot be evaluated by the SDK adapter." + def optimize( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + if _has_running_loop(): + raise ValueError( + "SDKBackend.optimize() cannot be called while an event loop is already running; " + "use await SDKBackend.optimize_async(...) instead." + ) + return asyncio.run( + self.optimize_async( + baseline_prompt=baseline_prompt, + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + output_dir=output_dir, + ) ) - def optimize( + async def optimize_async( self, *, baseline_prompt: str, @@ -91,35 +98,38 @@ def optimize( ) call_agent = _load_call_agent(self.call_agent_path) try: - from trpc_agent_sdk.evaluation import AgentEvaluator from trpc_agent_sdk.evaluation import AgentOptimizer from trpc_agent_sdk.evaluation import TargetPrompt except Exception as exc: # pragma: no cover - depends on optional SDK import health - raise ValueError(f"sdk mode could not import AgentEvaluator/AgentOptimizer/TargetPrompt: {exc}") from exc + raise ValueError(f"sdk mode could not import AgentOptimizer/TargetPrompt: {exc}") from exc - _ = AgentEvaluator target_prompt = TargetPrompt().add_path("system_prompt", str(self.prompt_path)) - result = asyncio.run( - AgentOptimizer.optimize( - config_path=str(optimizer_config_path), - call_agent=call_agent, - target_prompt=target_prompt, - train_dataset_path=str(train_path), - validation_dataset_path=str(val_path), - output_dir=str(output_dir), - update_source=self.update_source, - verbose=0, - ) + result = await AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target_prompt, + train_dataset_path=str(train_path), + validation_dataset_path=str(val_path), + output_dir=str(output_dir), + update_source=self.update_source, + verbose=0, ) best_prompt = getattr(result, "best_prompts", {}).get("system_prompt") if not best_prompt: raise ValueError("sdk mode completed but OptimizeResult.best_prompts['system_prompt'] was missing") + self.last_result_summary = _summarize_sdk_result(result) + self.last_artifact_dir = str(output_dir) return [ CandidatePrompt( candidate_id="sdk_best", prompt=best_prompt, rationale="Best prompt returned by AgentOptimizer.optimize.", - prompt_diff=_simple_diff(baseline_prompt, best_prompt), + prompt_diff=make_unified_diff( + baseline_prompt, + best_prompt, + before_name="baseline_system_prompt.txt", + after_name="sdk_best/system_prompt.txt", + ), ) ] @@ -128,14 +138,39 @@ def _load_call_agent(path: str): if ":" not in path: raise ValueError("--sdk-call-agent must use module:function format") module_name, function_name = path.split(":", 1) - module = importlib.import_module(module_name) + try: + module = importlib.import_module(module_name) + except Exception as exc: + raise ValueError(f"--sdk-call-agent target {path!r} could not import module {module_name!r}: {exc}") from exc call_agent = getattr(module, function_name, None) if call_agent is None: raise ValueError(f"--sdk-call-agent target {path!r} was not found") return call_agent -def _simple_diff(before: str, after: str) -> str: - if before == after: - return "# no prompt changes" - return "- baseline system_prompt\n+ optimized system_prompt" +def _has_running_loop() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +def _summarize_sdk_result(result: Any) -> dict[str, Any]: + if hasattr(result, "model_dump"): + payload = result.model_dump(mode="json") + elif hasattr(result, "__dict__"): + payload = dict(result.__dict__) + else: + payload = {"repr": repr(result)} + return _safe_jsonable(payload) + + +def _safe_jsonable(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _safe_jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_safe_jsonable(item) for item in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return repr(value) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/diffing.py b/examples/optimization/eval_optimize_loop/eval_loop/diffing.py new file mode 100644 index 00000000..9d578450 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/diffing.py @@ -0,0 +1,19 @@ +"""Prompt diff helpers.""" + +from __future__ import annotations + +import difflib + + +def make_unified_diff(before: str, after: str, *, before_name: str, after_name: str) -> str: + """Return a stable unified diff for prompt text.""" + + diff = difflib.unified_diff( + before.splitlines(), + after.splitlines(), + fromfile=before_name, + tofile=after_name, + lineterm="", + ) + rendered = "\n".join(line.rstrip() for line in diff) + return rendered or f"--- {before_name}\n+++ {after_name}\n# no prompt changes" diff --git a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py index ecb56391..da91ee12 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py @@ -2,6 +2,7 @@ from __future__ import annotations +from .diffing import make_unified_diff from .schemas import CandidatePrompt @@ -9,44 +10,49 @@ class FakeOptimizer: """Produce the two candidates required by the example issue.""" def propose(self, baseline_prompt: str) -> list[CandidatePrompt]: + overfit_prompt = ( + baseline_prompt.rstrip() + + "\n\n" + + "# Optimizer patch\n" + + "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n" + + "Always force every final answer into JSON, even when the user asks for prose.\n" + ) + safe_prompt = ( + baseline_prompt.rstrip() + + "\n\n" + + "# Optimizer patch\n" + + "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n" + + "Use strict JSON only when the user explicitly asks for JSON.\n" + + "Use exact answers only when the user explicitly asks for an exact answer.\n" + + "Otherwise answer naturally and honor rubric constraints.\n" + ) return [ CandidatePrompt( candidate_id="candidate_001_overfit", - prompt=( - baseline_prompt.rstrip() - + "\n\n" - + "# Optimizer patch\n" - + "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n" - + "Always force every final answer into JSON, even when the user asks for prose.\n" - ), + prompt=overfit_prompt, rationale=( "The train failures are strict JSON/exact formatting failures, so this candidate " "over-corrects by forcing JSON globally." ), - prompt_diff=( - "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n" - "+ Always force every final answer into JSON, even when the user asks for prose." + prompt_diff=make_unified_diff( + baseline_prompt, + overfit_prompt, + before_name="baseline_system_prompt.txt", + after_name="candidate_001_overfit/system_prompt.txt", ), ), CandidatePrompt( candidate_id="candidate_002_safe", - prompt=( - baseline_prompt.rstrip() - + "\n\n" - + "# Optimizer patch\n" - + "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n" - + "Use strict JSON only when the user explicitly asks for JSON.\n" - + "Use exact answers only when the user explicitly asks for an exact answer.\n" - + "Otherwise answer naturally and honor rubric constraints.\n" - ), + prompt=safe_prompt, rationale=( "This candidate fixes observed strict-format failures without changing " "natural-language behavior on validation cases." ), - prompt_diff=( - "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n" - "+ Use strict JSON only when explicitly requested.\n" - "+ Preserve natural-language answers unless a strict format is requested." + prompt_diff=make_unified_diff( + baseline_prompt, + safe_prompt, + before_name="baseline_system_prompt.txt", + after_name="candidate_002_safe/system_prompt.txt", ), ), ] diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index 11d18885..88df999c 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -132,7 +132,11 @@ def render_markdown(report: OptimizationReport) -> str: "", ]) for decision in report.gate_decisions: - verdict = "accepted" if decision.accepted else "rejected" + verdict = ( + decision.gate_status + if decision.gate_status != "applied" + else ("accepted" if decision.accepted else "rejected") + ) lines.append(f"### {decision.candidate_id} ({verdict})") for reason in decision.reasons: lines.append(f"- {reason}") @@ -148,7 +152,7 @@ def render_markdown(report: OptimizationReport) -> str: for record in report.candidates: candidate: CandidatePrompt = record["candidate"] gate = decision_by_id[candidate.candidate_id] - verdict = "accept" if gate.accepted else "reject" + verdict = gate.gate_status if gate.gate_status != "applied" else ("accept" if gate.accepted else "reject") lines.append( f"| {candidate.candidate_id} | {record['train_result'].score:.3f} | " f"{record['validation_result'].score:.3f} | {verdict} |" @@ -218,7 +222,9 @@ def render_markdown(report: OptimizationReport) -> str: "## Reproducibility", "", "```bash", - REPRODUCIBILITY_COMMAND, + report.run.get("reproducibility_command") + or report.audit.get("reproducibility_command") + or REPRODUCIBILITY_COMMAND, "```", "", ]) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py index b6712233..4e72c6ea 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -120,6 +120,8 @@ class GateDecision: cumulative_cost: float total_run_cost: float cost: float + gate_status: str = "applied" + gate_not_applied_reason: str | None = None @dataclass(frozen=True) diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json index 2d7318d9..b6ecb44e 100644 --- a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json @@ -7,12 +7,12 @@ "candidate_prompts": { "candidate_001_overfit": { "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", - "prompt_diff": "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+ Always force every final answer into JSON, even when the user asks for prose.", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", "rationale": "The train failures are strict JSON/exact formatting failures, so this candidate over-corrects by forcing JSON globally." }, "candidate_002_safe": { "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\nUse strict JSON only when the user explicitly asks for JSON.\nUse exact answers only when the user explicitly asks for an exact answer.\nOtherwise answer naturally and honor rubric constraints.\n", - "prompt_diff": "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+ Use strict JSON only when explicitly requested.\n+ Preserve natural-language answers unless a strict format is requested.", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints.", "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." } }, @@ -39,11 +39,13 @@ "validation": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\val.evalset.json" }, "prompt_diffs": { - "candidate_001_overfit": "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+ Always force every final answer into JSON, even when the user asks for prose.", - "candidate_002_safe": "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+ Use strict JSON only when explicitly requested.\n+ Preserve natural-language answers unless a strict format is requested." + "candidate_001_overfit": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", + "candidate_002_safe": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints." }, + "prompt_hash": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95", "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", - "seed": 91 + "seed": 91, + "total_run_cost": 0.018 }, "baseline": { "train": { @@ -432,7 +434,7 @@ "candidate": { "candidate_id": "candidate_001_overfit", "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", - "prompt_diff": "+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+ Always force every final answer into JSON, even when the user asks for prose.", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", "rationale": "The train failures are strict JSON/exact formatting failures, so this candidate over-corrects by forcing JSON globally." }, "train_result": { @@ -631,7 +633,7 @@ "candidate": { "candidate_id": "candidate_002_safe", "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\nUse strict JSON only when the user explicitly asks for JSON.\nUse exact answers only when the user explicitly asks for an exact answer.\nOtherwise answer naturally and honor rubric constraints.\n", - "prompt_diff": "+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+ Use strict JSON only when explicitly requested.\n+ Preserve natural-language answers unless a strict format is requested.", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints.", "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." }, "train_result": { @@ -1052,6 +1054,8 @@ "val_explain_cache", "val_protected_yes_no" ], + "gate_not_applied_reason": null, + "gate_status": "applied", "new_hard_failures": [ "val_explain_cache", "val_protected_yes_no" @@ -1082,6 +1086,8 @@ "cost": 0.006, "cumulative_cost": 0.012, "excessive_score_drops": [], + "gate_not_applied_reason": null, + "gate_status": "applied", "new_hard_failures": [], "overfit_detected": false, "protected_regressions": [], @@ -1251,6 +1257,7 @@ "validation": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\val.evalset.json" }, "prompt_source": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", + "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", "run_id": "eval_optimize_loop_seed_91", "trace_enabled": true, "train_cases": 3, diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md index a4e8e862..510c7f65 100644 --- a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md @@ -66,16 +66,33 @@ Run id: `eval_optimize_loop_seed_91` ### candidate_001_overfit ```diff -+ OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON -+ Always force every final answer into JSON, even when the user asks for prose. +--- baseline_system_prompt.txt ++++ candidate_001_overfit/system_prompt.txt +@@ -2,3 +2,7 @@ + + Answer clearly and include a short explanation when it may help the user. + If the user asks for structured data, provide the information they need. ++ ++# Optimizer patch ++OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON ++Always force every final answer into JSON, even when the user asks for prose. ``` ### candidate_002_safe ```diff -+ OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED -+ Use strict JSON only when explicitly requested. -+ Preserve natural-language answers unless a strict format is requested. +--- baseline_system_prompt.txt ++++ candidate_002_safe/system_prompt.txt +@@ -2,3 +2,9 @@ + + Answer clearly and include a short explanation when it may help the user. + If the user asks for structured data, provide the information they need. ++ ++# Optimizer patch ++OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED ++Use strict JSON only when the user explicitly asks for JSON. ++Use exact answers only when the user explicitly asks for an exact answer. ++Otherwise answer naturally and honor rubric constraints. ``` ## Reproducibility diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 17e704be..06aa2b0b 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -4,6 +4,7 @@ import argparse import hashlib +import json import sys import tempfile from pathlib import Path @@ -27,6 +28,8 @@ from eval_loop.report import compute_case_deltas from eval_loop.report import write_reports from eval_loop.schemas import CandidatePrompt +from eval_loop.schemas import EvalResult +from eval_loop.schemas import GateDecision from eval_loop.schemas import OptimizationReport @@ -61,34 +64,49 @@ def run_pipeline( "or use --mode sdk with --sdk-call-agent module:function." ) - train_cases = load_eval_cases(train_path, split="train") - validation_cases = load_eval_cases(val_path, split="validation") - optimizer_config = load_optimizer_config(optimizer_config_path) - validate_inputs( - train_path=train_path, - val_path=val_path, - optimizer_config_path=optimizer_config_path, - train_cases=train_cases, - validation_cases=validation_cases, - config=optimizer_config, - ) if mode == "sdk": - SDKBackend( + optimizer_config_dict = _read_json_object_for_audit(optimizer_config_path) + baseline_prompt = load_prompt(prompt_path) + sdk_backend = SDKBackend( prompt_path=prompt_path, call_agent_path=sdk_call_agent, update_source=update_source, - ).optimize( - baseline_prompt=load_prompt(prompt_path), + ) + candidates = sdk_backend.optimize( + baseline_prompt=baseline_prompt, train_path=train_path, val_path=val_path, optimizer_config_path=optimizer_config_path, output_dir=output_dir, ) - raise ValueError( - "sdk mode delegated optimization to AgentOptimizer. Inspect the SDK artifacts in output_dir; " - "the example JSON/Markdown report is generated by fake mode. To run offline review artifacts, " - "use --mode fake --trace." + report = _build_sdk_report( + candidates=candidates, + sdk_backend=sdk_backend, + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=output_dir, + trace=trace, + update_source=update_source, + train_case_count=_count_cases(train_path), + validation_case_count=_count_cases(val_path), + optimizer_config_dict=optimizer_config_dict, ) + write_reports(report, output_dir) + return report + + optimizer_config = load_optimizer_config(optimizer_config_path) + train_cases = load_eval_cases(train_path, split="train") + validation_cases = load_eval_cases(val_path, split="validation") + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + train_cases=train_cases, + validation_cases=validation_cases, + config=optimizer_config, + ) seed = optimizer_config.seed baseline_prompt = load_prompt(prompt_path) @@ -195,6 +213,7 @@ def run_pipeline( "train_cases": len(train_cases), "validation_cases": len(validation_cases), "update_source": update_source, + "reproducibility_command": REPRODUCIBILITY_COMMAND, "paths": { "train": str(train_path), "validation": str(val_path), @@ -265,7 +284,9 @@ def _build_audit( "config_hash": config_hash, "input_hashes": input_hashes, "input_paths": input_paths, + "prompt_hash": input_hashes["prompt"], "candidate_prompt_hashes": candidate_prompt_hashes, + "total_run_cost": total_cost, "cost": { "baseline": baseline_cost, "candidates": candidate_costs, @@ -287,6 +308,172 @@ def _build_audit( } +def _build_sdk_report( + *, + candidates: list[CandidatePrompt], + sdk_backend: SDKBackend, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + prompt_path: str | Path, + output_dir: str | Path, + trace: bool, + update_source: bool, + train_case_count: int | None, + validation_case_count: int | None, + optimizer_config_dict: dict[str, Any], +) -> OptimizationReport: + input_hashes = _input_hashes( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + ) + baseline_train = EvalResult(prompt_id="baseline", split="train", score=0.0, passed=False, cost=0.0, cases=[]) + baseline_validation = EvalResult( + prompt_id="baseline", + split="validation", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ) + candidate_records = [ + { + "candidate": candidate, + "train_result": EvalResult( + prompt_id=candidate.candidate_id, + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ), + "validation_result": EvalResult( + prompt_id=candidate.candidate_id, + split="validation", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ), + "gate_status": "not_applicable", + "gate_not_applied_reason": "SDK optimizer result does not expose per-case validation results", + "sdk_result_summary": sdk_backend.last_result_summary or {}, + } + for candidate in candidates + ] + prompt_hashes = { + candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() + for candidate in candidates + } + audit = { + "seed": None, + "duration_seconds": 0.0, + "config_hash": stable_config_hash(optimizer_config_dict), + "input_hashes": input_hashes, + "input_paths": { + "train": str(train_path), + "validation": str(val_path), + "optimizer": str(optimizer_config_path), + "prompt": str(prompt_path), + }, + "prompt_hash": input_hashes["prompt"], + "candidate_prompt_hashes": prompt_hashes, + "total_run_cost": 0.0, + "cost": {"baseline": 0.0, "candidates": {candidate.candidate_id: 0.0 for candidate in candidates}, "total": 0.0}, + "candidate_prompts": { + candidate.candidate_id: { + "rationale": candidate.rationale, + "prompt": candidate.prompt, + "prompt_diff": candidate.prompt_diff, + } + for candidate in candidates + }, + "prompt_diffs": {candidate.candidate_id: candidate.prompt_diff for candidate in candidates}, + "sdk_artifact_dir": str(output_dir), + "sdk_result_summary": sdk_backend.last_result_summary or {}, + "reproducibility_command": _sdk_reproducibility_command( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=output_dir, + update_source=update_source, + ), + } + run = { + "run_id": "eval_optimize_loop_sdk", + "mode": "sdk", + "fake_model": False, + "fake_judge": False, + "trace_enabled": trace, + "train_cases": train_case_count, + "validation_cases": validation_case_count, + "update_source": update_source, + "sdk_artifact_dir": str(output_dir), + "reproducibility_command": audit["reproducibility_command"], + "paths": { + "train": str(train_path), + "validation": str(val_path), + "optimizer": str(optimizer_config_path), + "prompt": str(prompt_path), + }, + "prompt_source": str(prompt_path), + } + gate_decisions = [ + GateDecision( + candidate_id=candidate.candidate_id, + accepted=True, + reasons=["gate not applied: SDK optimizer result does not expose per-case validation results"], + train_score_delta=0.0, + validation_score_delta=0.0, + new_hard_failures=[], + protected_regressions=[], + validation_new_failures=[], + excessive_score_drops=[], + overfit_detected=False, + candidate_cost=0.0, + cumulative_cost=0.0, + total_run_cost=0.0, + cost=0.0, + gate_status="not_applicable", + gate_not_applied_reason="SDK optimizer result does not expose per-case validation results", + ) + for candidate in candidates + ] + return build_report( + run=run, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + per_case_deltas=[], + gate_decisions=gate_decisions, + selected_candidate=candidates[0].candidate_id if candidates else None, + audit=audit, + ) + + +def _sdk_reproducibility_command( + *, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + prompt_path: str | Path, + output_dir: str | Path, + update_source: bool, +) -> str: + command = ( + "python examples/optimization/eval_optimize_loop/run_pipeline.py " + f"--mode sdk --train {train_path} --val {val_path} " + f"--optimizer-config {optimizer_config_path} --prompt {prompt_path} " + f"--output-dir {output_dir} --sdk-call-agent module:function" + ) + if update_source: + command += " --update-source" + return command + + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--train", default=str(DEFAULT_TRAIN), help="Path to train.evalset.json") @@ -318,6 +505,25 @@ def _input_hashes( } +def _count_cases(path: str | Path) -> int | None: + try: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + except Exception: + return None + for key in ("cases", "eval_cases"): + cases = payload.get(key) if isinstance(payload, dict) else None + if isinstance(cases, list): + return len(cases) + return None + + +def _read_json_object_for_audit(path: str | Path) -> dict[str, Any]: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{path}: optimizer config must be a JSON object") + return payload + + def main(argv: list[str] | None = None) -> OptimizationReport: args = parse_args(argv) report = run_pipeline( diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py index 40b5adfb..fc112caf 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_gate.py +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -165,6 +165,7 @@ def test_gate_rejects_cost_budget(): assert not decision.accepted assert decision.total_run_cost > 0.001 + assert decision.total_run_cost == decision.cumulative_cost + decision.candidate_cost def test_case_delta_types_are_classified(): diff --git a/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py new file mode 100644 index 00000000..318dd460 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + + +BUSINESS_LOGIC_FILES = [ + "eval_loop/fake_model.py", + "eval_loop/fake_judge.py", + "eval_loop/optimizer.py", + "eval_loop/report.py", + "eval_loop/gate.py", +] + + +def test_sample_case_ids_do_not_appear_in_business_logic(): + root = Path("examples/optimization/eval_optimize_loop") + case_ids = set() + for rel in ("data/train.evalset.json", "data/val.evalset.json"): + payload = json.loads((root / rel).read_text(encoding="utf-8")) + case_ids.update(case["id"] for case in payload["cases"]) + + for rel in BUSINESS_LOGIC_FILES: + source = (root / rel).read_text(encoding="utf-8") + leaked = sorted(case_id for case_id in case_ids if case_id in source) + assert leaked == [], f"{rel} contains sample case ids: {leaked}" + + +def test_uuid_style_case_ids_still_drive_expected_behavior(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + optimizer_path = tmp_path / "optimizer.json" + prompt_path = tmp_path / "prompt.txt" + + train_path.write_text( + json.dumps({ + "split": "train", + "cases": [ + _json_case("1d5b548a-39ec-451d-9b01-7d73101b95b1", "train"), + _exact_case("c28596bf-370d-4898-bdf7-e4cc306fa4d3", "train", protected=False), + _rubric_case("3aa39806-878e-4634-9a68-d7f479ab7c9d", "train"), + ], + }), + encoding="utf-8", + ) + val_path.write_text( + json.dumps({ + "split": "validation", + "cases": [ + _json_case("534f045c-06f8-4b96-af55-cb7b6712cc0c", "validation"), + _rubric_case("b6914390-b2bb-4f43-a35b-24b6fe2825cd", "validation"), + _exact_case("70c59d31-adf5-409f-a457-286bcb887f52", "validation", protected=True), + ], + }), + encoding="utf-8", + ) + optimizer_path.write_text( + json.dumps({ + "seed": 91, + "optimizer": {"name": "fake_two_candidate_optimizer"}, + "metrics": {"case_score": "mean"}, + "gate": { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": ["70c59d31-adf5-409f-a457-286bcb887f52"], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, + }, + }), + encoding="utf-8", + ) + prompt_path.write_text("Baseline prompt", encoding="utf-8") + + report = run_pipeline( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + trace=True, + ) + + decisions = {decision.candidate_id: decision for decision in report.gate_decisions} + assert report.selected_candidate == "candidate_002_safe" + assert decisions["candidate_001_overfit"].accepted is False + assert decisions["candidate_001_overfit"].overfit_detected is True + assert decisions["candidate_002_safe"].accepted is True + + +def _json_case(case_id: str, split: str) -> dict: + return { + "id": case_id, + "input": "Return strict JSON.", + "expectation": { + "type": "json", + "required_keys": ["answer"], + "expected_values": {"answer": "ok"}, + "expected_failure_category": "format_violation", + }, + } + + +def _exact_case(case_id: str, split: str, *, protected: bool) -> dict: + payload = { + "id": case_id, + "input": "Answer exactly YES.", + "expectation": { + "type": "exact", + "expected": "YES", + "expected_failure_category": "final_response_mismatch", + }, + "protected": protected, + "tags": ["baseline_pass"] if protected else [], + } + return payload + + +def _rubric_case(case_id: str, split: str) -> dict: + return { + "id": case_id, + "input": "Explain in prose.", + "expectation": { + "type": "rubric", + "must_include": ["cache", "stale data"], + "forbidden": ["{", "}", "json"], + "max_chars": 120, + "expected_failure_category": "format_violation", + }, + "tags": ["prose"], + } diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py index 6a6ce68c..d43810f8 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -61,17 +61,23 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): "config_hash", "cost", "duration_seconds", + "prompt_hash", "candidate_prompts", "prompt_diffs", "input_hashes", "candidate_prompt_hashes", + "total_run_cost", } + assert payload["run"]["reproducibility_command"].startswith("python examples/optimization") + assert payload["audit"]["total_run_cost"] == payload["audit"]["cost"]["total"] assert "candidate_001_overfit" in payload["audit"]["prompt_diffs"] assert "candidate_002_safe" in payload["audit"]["prompt_diffs"] + assert payload["candidates"][0]["candidate"]["prompt_diff"].startswith("--- baseline_system_prompt.txt") decisions = {item["candidate_id"]: item for item in payload["gate_decisions"]} assert not decisions["candidate_001_overfit"]["accepted"] assert decisions["candidate_002_safe"]["accepted"] + assert decisions["candidate_001_overfit"]["total_run_cost"] > decisions["candidate_001_overfit"]["candidate_cost"] assert any( "train score improved but validation score regressed" in reason for reason in decisions["candidate_001_overfit"]["reasons"] @@ -94,6 +100,9 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert (run_dir / "candidate_prompts" / "candidate_001_overfit" / "system_prompt.txt").is_file() assert (run_dir / "case_results" / "candidate_002_safe_validation.json").is_file() assert (run_dir / "prompt_diffs" / "candidate_002_safe.diff").is_file() + assert (run_dir / "prompt_diffs" / "candidate_002_safe.diff").read_text(encoding="utf-8") == ( + payload["candidates"][1]["candidate"]["prompt_diff"] + ) def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index a4be6eab..56d8afbc 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -1,5 +1,7 @@ from __future__ import annotations +import builtins +import json import sys import types from pathlib import Path @@ -7,6 +9,11 @@ import pytest from examples.optimization.eval_optimize_loop.eval_loop.backends import SDKBackend +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline def test_sdk_backend_requires_call_agent_path(tmp_path: Path): @@ -23,6 +30,226 @@ def test_sdk_backend_requires_call_agent_path(tmp_path: Path): def test_sdk_backend_calls_agent_optimizer_and_converts_best_prompt(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + candidates = backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["config_path"].endswith("optimizer.json") + assert calls["update_source"] is False + assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] + assert candidates[0].candidate_id == "sdk_best" + assert candidates[0].prompt == "optimized prompt" + assert candidates[0].prompt_diff.startswith("--- baseline_system_prompt.txt") + + +def test_sdk_backend_passes_update_source_true(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + update_source=True, + ).optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["update_source"] is True + + +def test_sdk_backend_call_agent_import_failure_names_target(tmp_path: Path): + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="missing.module:call_agent") + + with pytest.raises(ValueError, match="missing.module:call_agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_sdk_import_failure_is_clear(tmp_path: Path, monkeypatch): + call_agent_module = types.ModuleType("fake_call_agent_module") + + async def call_agent(query: str) -> str: + return query + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + + real_import = builtins.__import__ + + def fail_sdk_import(name, globals=None, locals=None, fromlist=(), level=0): + if name == "trpc_agent_sdk.evaluation": + raise ImportError("forced sdk import failure") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", fail_sdk_import) + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="AgentOptimizer/TargetPrompt"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_empty_best_prompt_error_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="best_prompts\\['system_prompt'\\]"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +@pytest.mark.asyncio +async def test_sdk_backend_sync_optimize_rejects_active_event_loop(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="optimize_async"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +@pytest.mark.asyncio +async def test_sdk_backend_async_api_works_inside_active_event_loop(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + candidates = await backend.optimize_async( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert candidates[0].candidate_id == "sdk_best" + + +def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + output_dir = tmp_path / "sdk_run" + payload = (output_dir / "optimization_report.json").read_text(encoding="utf-8") + markdown = (output_dir / "optimization_report.md").read_text(encoding="utf-8") + assert report.run["mode"] == "sdk" + assert report.run["update_source"] is False + assert report.selected_candidate == "sdk_best" + assert report.gate_decisions[0].gate_status == "not_applicable" + assert "SDK optimizer result does not expose per-case validation results" in payload + assert "sdk_best (not_applicable)" in markdown + assert (output_dir / "runs" / "eval_optimize_loop_sdk" / "input_hashes.json").is_file() + assert (output_dir / "runs" / "eval_optimize_loop_sdk" / "prompt_diffs" / "sdk_best.diff").is_file() + assert calls["update_source"] is False + + +def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + train_path = tmp_path / "sdk_train.evalset.json" + val_path = tmp_path / "sdk_val.evalset.json" + optimizer_path = tmp_path / "sdk_optimizer.json" + prompt_path = tmp_path / "system_prompt.txt" + train_path.write_text(json.dumps({"eval_cases": []}), encoding="utf-8") + val_path.write_text(json.dumps({"eval_cases": []}), encoding="utf-8") + optimizer_path.write_text( + json.dumps({"seed": "sdk-owned-seed", "optimize": {"algorithm": {"name": "gepa_reflective"}}}), + encoding="utf-8", + ) + prompt_path.write_text("baseline", encoding="utf-8") + + report = run_pipeline( + mode="sdk", + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + assert report.run["mode"] == "sdk" + assert report.run["train_cases"] == 0 + assert report.selected_candidate == "sdk_best" + + +def test_run_pipeline_mode_sdk_passes_update_source_true(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + update_source=True, + ) + + assert report.run["update_source"] is True + assert calls["update_source"] is True + + +def test_run_pipeline_mode_sdk_missing_call_agent_is_not_fake_fallback(tmp_path: Path): + with pytest.raises(ValueError, match="--sdk-call-agent"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + ) + + +def _install_fake_sdk(monkeypatch, *, best_prompt: str): calls = {} class FakeTargetPrompt: @@ -37,10 +264,13 @@ class FakeAgentOptimizer: @staticmethod async def optimize(**kwargs): calls.update(kwargs) - return types.SimpleNamespace(best_prompts={"system_prompt": "optimized prompt"}) + return types.SimpleNamespace( + best_prompts={"system_prompt": best_prompt}, + status="SUCCEEDED", + best_pass_rate=1.0, + ) fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") - fake_eval_module.AgentEvaluator = object fake_eval_module.AgentOptimizer = FakeAgentOptimizer fake_eval_module.TargetPrompt = FakeTargetPrompt monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) @@ -52,21 +282,4 @@ async def call_agent(query: str) -> str: call_agent_module.call_agent = call_agent monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) - - prompt_path = tmp_path / "prompt.txt" - prompt_path.write_text("baseline", encoding="utf-8") - backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") - - candidates = backend.optimize( - baseline_prompt="baseline", - train_path=tmp_path / "train.evalset.json", - val_path=tmp_path / "val.evalset.json", - optimizer_config_path=tmp_path / "optimizer.json", - output_dir=tmp_path / "out", - ) - - assert calls["config_path"].endswith("optimizer.json") - assert calls["update_source"] is False - assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] - assert candidates[0].candidate_id == "sdk_best" - assert candidates[0].prompt == "optimized prompt" + return calls From 680369cc9fda496c9a54c5955df637267bb4901d Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 21:24:04 +0800 Subject: [PATCH 05/34] Map SDK optimize aggregates into eval report --- .../optimization/eval_optimize_loop/README.md | 15 +- .../eval_optimize_loop/eval_loop/backends.py | 37 +++- .../eval_optimize_loop/eval_loop/report.py | 2 + .../eval_optimize_loop/eval_loop/schemas.py | 1 + .../outputs/optimization_report.example.json | 2 + .../eval_optimize_loop/run_pipeline.py | 165 ++++++++++++++---- .../tests/test_sdk_backend.py | 126 ++++++++++++- 7 files changed, 298 insertions(+), 50 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index f0fdc43a..8c03b074 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -69,10 +69,14 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ `--sdk-call-agent` must point to an async callable compatible with `AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials needed by that callable. SDK mode never silently falls back to fake mode. When -the SDK optimizer returns a best prompt but does not expose per-case validation -results, this example still writes JSON/Markdown reports and audit artifacts -with `gate_status: not_applicable`, records the SDK result summary, and selects -`sdk_best` as the SDK optimizer's chosen prompt. +the SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult` +aggregate fields into the JSON/Markdown report: baseline/best pass rate, +pass-rate improvement, metric breakdowns, token usage, duration, LLM cost, and +round summaries. SDK mode applies an aggregate gate with +`gate_status: partial_applied`: it checks `status == SUCCEEDED`, validation +improvement against `gate.min_val_score_improvement`, and total LLM cost against +`gate.max_total_cost`. Per-case checks that require full validation case scores +are listed in `not_applied_checks` instead of being silently treated as passed. ## Source Prompt Writes @@ -110,7 +114,8 @@ optional `simulated_outputs`; it does not depend on sample `case_id` names. - failure attribution summary and attribution accuracy when expected labels are present; - gate decisions with overfit detection, protected regressions, new hard - failures, excessive drops, and cost fields; + failures, excessive drops, cost fields, and SDK `not_applied_checks` when + per-case validation details are not exposed; - audit data: seed, duration, config hash, input hashes, candidate prompt hashes, cost, prompt diffs, and reproducibility command. diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 1e11ff41..3dec6e8b 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -54,6 +54,7 @@ class SDKBackend: prompt_path: str | Path call_agent_path: str | None = None update_source: bool = False + last_result: Any | None = None last_result_summary: dict[str, Any] | None = None last_artifact_dir: str | None = None @@ -117,6 +118,7 @@ async def optimize_async( best_prompt = getattr(result, "best_prompts", {}).get("system_prompt") if not best_prompt: raise ValueError("sdk mode completed but OptimizeResult.best_prompts['system_prompt'] was missing") + self.last_result = result self.last_result_summary = _summarize_sdk_result(result) self.last_artifact_dir = str(output_dir) return [ @@ -157,16 +159,37 @@ def _has_running_loop() -> bool: def _summarize_sdk_result(result: Any) -> dict[str, Any]: - if hasattr(result, "model_dump"): - payload = result.model_dump(mode="json") - elif hasattr(result, "__dict__"): - payload = dict(result.__dict__) - else: - payload = {"repr": repr(result)} - return _safe_jsonable(payload) + return { + "status": _safe_jsonable(getattr(result, "status", None)), + "baseline_pass_rate": _safe_jsonable(getattr(result, "baseline_pass_rate", None)), + "best_pass_rate": _safe_jsonable(getattr(result, "best_pass_rate", None)), + "pass_rate_improvement": _safe_jsonable(getattr(result, "pass_rate_improvement", None)), + "baseline_metric_breakdown": _safe_jsonable(getattr(result, "baseline_metric_breakdown", {})), + "best_metric_breakdown": _safe_jsonable(getattr(result, "best_metric_breakdown", {})), + "metric_thresholds": _safe_jsonable(getattr(result, "metric_thresholds", {})), + "total_llm_cost": _safe_jsonable(getattr(result, "total_llm_cost", 0.0)), + "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})), + "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)), + "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)), + "rounds": [ + { + "validation_pass_rate": _safe_jsonable(getattr(round_record, "validation_pass_rate", None)), + "accepted": _safe_jsonable(getattr(round_record, "accepted", None)), + "failed_case_ids": _safe_jsonable(getattr(round_record, "failed_case_ids", [])), + "round_llm_cost": _safe_jsonable(getattr(round_record, "round_llm_cost", 0.0)), + "budget_used": _safe_jsonable(getattr(round_record, "budget_used", None)), + "budget_total": _safe_jsonable(getattr(round_record, "budget_total", None)), + } + for round_record in getattr(result, "rounds", []) or [] + ], + } def _safe_jsonable(value: Any) -> Any: + if hasattr(value, "model_dump"): + return _safe_jsonable(value.model_dump(mode="json")) + if hasattr(value, "__dict__") and not isinstance(value, type): + return _safe_jsonable(dict(value.__dict__)) if isinstance(value, dict): return {str(key): _safe_jsonable(item) for key, item in value.items()} if isinstance(value, (list, tuple)): diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index 88df999c..3b264f4a 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -140,6 +140,8 @@ def render_markdown(report: OptimizationReport) -> str: lines.append(f"### {decision.candidate_id} ({verdict})") for reason in decision.reasons: lines.append(f"- {reason}") + if decision.not_applied_checks: + lines.append(f"- not applied checks: {', '.join(decision.not_applied_checks)}") lines.append("") lines.extend([ diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py index 4e72c6ea..c1ce402d 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -122,6 +122,7 @@ class GateDecision: cost: float gate_status: str = "applied" gate_not_applied_reason: str | None = None + not_applied_checks: list[str] = field(default_factory=list) @dataclass(frozen=True) diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json index b6ecb44e..4b41a37e 100644 --- a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json @@ -1060,6 +1060,7 @@ "val_explain_cache", "val_protected_yes_no" ], + "not_applied_checks": [], "overfit_detected": true, "protected_regressions": [ "val_protected_yes_no" @@ -1089,6 +1090,7 @@ "gate_not_applied_reason": null, "gate_status": "applied", "new_hard_failures": [], + "not_applied_checks": [], "overfit_detected": false, "protected_regressions": [], "reasons": [ diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 06aa2b0b..6a0c96e4 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -67,6 +67,7 @@ def run_pipeline( if mode == "sdk": optimizer_config_dict = _read_json_object_for_audit(optimizer_config_path) baseline_prompt = load_prompt(prompt_path) + sdk_artifact_dir = Path(output_dir) / "sdk_optimizer" sdk_backend = SDKBackend( prompt_path=prompt_path, call_agent_path=sdk_call_agent, @@ -77,7 +78,7 @@ def run_pipeline( train_path=train_path, val_path=val_path, optimizer_config_path=optimizer_config_path, - output_dir=output_dir, + output_dir=sdk_artifact_dir, ) report = _build_sdk_report( candidates=candidates, @@ -329,12 +330,24 @@ def _build_sdk_report( optimizer_config_path=optimizer_config_path, prompt_path=prompt_path, ) + sdk_summary = sdk_backend.last_result_summary or {} + baseline_pass_rate = _summary_float(sdk_summary, "baseline_pass_rate", 0.0) + best_pass_rate = _summary_float(sdk_summary, "best_pass_rate", baseline_pass_rate) + pass_rate_improvement = _summary_float( + sdk_summary, + "pass_rate_improvement", + best_pass_rate - baseline_pass_rate, + ) + total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0) + duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0) + gate_config = _sdk_gate_config(optimizer_config_dict) + baseline_train = EvalResult(prompt_id="baseline", split="train", score=0.0, passed=False, cost=0.0, cases=[]) baseline_validation = EvalResult( prompt_id="baseline", split="validation", - score=0.0, - passed=False, + score=baseline_pass_rate, + passed=baseline_pass_rate >= 1.0, cost=0.0, cases=[], ) @@ -352,14 +365,14 @@ def _build_sdk_report( "validation_result": EvalResult( prompt_id=candidate.candidate_id, split="validation", - score=0.0, - passed=False, - cost=0.0, + score=best_pass_rate, + passed=best_pass_rate >= baseline_pass_rate, + cost=total_llm_cost, cases=[], ), - "gate_status": "not_applicable", - "gate_not_applied_reason": "SDK optimizer result does not expose per-case validation results", - "sdk_result_summary": sdk_backend.last_result_summary or {}, + "gate_status": "partial_applied", + "gate_not_applied_reason": "SDK OptimizeResult exposes aggregate scores but not full per-case deltas", + "sdk_result_summary": sdk_summary, } for candidate in candidates ] @@ -369,7 +382,7 @@ def _build_sdk_report( } audit = { "seed": None, - "duration_seconds": 0.0, + "duration_seconds": duration_seconds, "config_hash": stable_config_hash(optimizer_config_dict), "input_hashes": input_hashes, "input_paths": { @@ -380,8 +393,12 @@ def _build_sdk_report( }, "prompt_hash": input_hashes["prompt"], "candidate_prompt_hashes": prompt_hashes, - "total_run_cost": 0.0, - "cost": {"baseline": 0.0, "candidates": {candidate.candidate_id: 0.0 for candidate in candidates}, "total": 0.0}, + "total_run_cost": total_llm_cost, + "cost": { + "baseline": 0.0, + "candidates": {candidate.candidate_id: total_llm_cost for candidate in candidates}, + "total": total_llm_cost, + }, "candidate_prompts": { candidate.candidate_id: { "rationale": candidate.rationale, @@ -391,8 +408,8 @@ def _build_sdk_report( for candidate in candidates }, "prompt_diffs": {candidate.candidate_id: candidate.prompt_diff for candidate in candidates}, - "sdk_artifact_dir": str(output_dir), - "sdk_result_summary": sdk_backend.last_result_summary or {}, + "sdk_artifact_dir": sdk_backend.last_artifact_dir or str(Path(output_dir) / "sdk_optimizer"), + "sdk_result_summary": sdk_summary, "reproducibility_command": _sdk_reproducibility_command( train_path=train_path, val_path=val_path, @@ -411,7 +428,7 @@ def _build_sdk_report( "train_cases": train_case_count, "validation_cases": validation_case_count, "update_source": update_source, - "sdk_artifact_dir": str(output_dir), + "sdk_artifact_dir": audit["sdk_artifact_dir"], "reproducibility_command": audit["reproducibility_command"], "paths": { "train": str(train_path), @@ -422,26 +439,16 @@ def _build_sdk_report( "prompt_source": str(prompt_path), } gate_decisions = [ - GateDecision( + _sdk_gate_decision( candidate_id=candidate.candidate_id, - accepted=True, - reasons=["gate not applied: SDK optimizer result does not expose per-case validation results"], - train_score_delta=0.0, - validation_score_delta=0.0, - new_hard_failures=[], - protected_regressions=[], - validation_new_failures=[], - excessive_score_drops=[], - overfit_detected=False, - candidate_cost=0.0, - cumulative_cost=0.0, - total_run_cost=0.0, - cost=0.0, - gate_status="not_applicable", - gate_not_applied_reason="SDK optimizer result does not expose per-case validation results", + sdk_summary=sdk_summary, + gate_config=gate_config, ) for candidate in candidates ] + selected_candidate = None + if candidates and gate_decisions and gate_decisions[0].accepted: + selected_candidate = candidates[0].candidate_id return build_report( run=run, baseline_train=baseline_train, @@ -449,7 +456,7 @@ def _build_sdk_report( candidate_records=candidate_records, per_case_deltas=[], gate_decisions=gate_decisions, - selected_candidate=candidates[0].candidate_id if candidates else None, + selected_candidate=selected_candidate, audit=audit, ) @@ -474,6 +481,100 @@ def _sdk_reproducibility_command( return command +def _sdk_gate_decision( + *, + candidate_id: str, + sdk_summary: dict[str, Any], + gate_config: dict[str, float], +) -> GateDecision: + status = str(sdk_summary.get("status") or "UNKNOWN") + improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0) + total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0) + min_improvement = gate_config["min_val_score_improvement"] + max_cost = gate_config["max_total_cost"] + + reasons: list[str] = [] + accepted = True + if status != "SUCCEEDED": + accepted = False + reasons.append(f"reject: SDK optimizer status {status} is not SUCCEEDED") + else: + reasons.append("accept: SDK optimizer status is SUCCEEDED") + + if improvement < min_improvement: + accepted = False + reasons.append( + f"reject: validation improvement {improvement:.3f} is below threshold {min_improvement:.3f}" + ) + else: + reasons.append( + f"accept: validation improvement {improvement:.3f} meets threshold {min_improvement:.3f}" + ) + + if total_cost > max_cost: + accepted = False + reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}") + else: + reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}") + + if accepted: + reasons.append("accept: SDK aggregate gate passed") + + return GateDecision( + candidate_id=candidate_id, + accepted=accepted, + reasons=reasons, + train_score_delta=0.0, + validation_score_delta=improvement, + new_hard_failures=[], + protected_regressions=[], + validation_new_failures=[], + excessive_score_drops=[], + overfit_detected=False, + candidate_cost=total_cost, + cumulative_cost=0.0, + total_run_cost=total_cost, + cost=total_cost, + gate_status="partial_applied", + gate_not_applied_reason="SDK OptimizeResult exposes aggregate scores but not full per-case deltas", + not_applied_checks=[ + "per_case_delta", + "protected_regression", + "new_hard_failure", + "max_score_drop_per_case", + ], + ) + + +def _sdk_gate_config(optimizer_config_dict: dict[str, Any]) -> dict[str, float]: + gate_payload = optimizer_config_dict.get("gate") + if gate_payload is None: + gate_payload = {} + if not isinstance(gate_payload, dict): + raise ValueError("optimizer config field 'gate' must be an object when present") + + min_improvement = gate_payload.get("min_val_score_improvement", 0.01) + max_cost = gate_payload.get("max_total_cost", 1.0) + if not isinstance(min_improvement, (int, float)) or min_improvement < 0: + raise ValueError("optimizer config field 'gate.min_val_score_improvement' must be a non-negative number") + if not isinstance(max_cost, (int, float)) or max_cost < 0: + raise ValueError("optimizer config field 'gate.max_total_cost' must be a non-negative number") + return { + "min_val_score_improvement": float(min_improvement), + "max_total_cost": float(max_cost), + } + + +def _summary_float(summary: dict[str, Any], key: str, default: float) -> float: + value = summary.get(key, default) + if value is None: + return default + try: + return float(value) + except (TypeError, ValueError): + return default + + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--train", default=str(DEFAULT_TRAIN), help="Path to train.evalset.json") diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 56d8afbc..8b0c1edf 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -46,10 +46,13 @@ def test_sdk_backend_calls_agent_optimizer_and_converts_best_prompt(tmp_path: Pa assert calls["config_path"].endswith("optimizer.json") assert calls["update_source"] is False + assert calls["output_dir"].endswith("out") assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] assert candidates[0].candidate_id == "sdk_best" assert candidates[0].prompt == "optimized prompt" assert candidates[0].prompt_diff.startswith("--- baseline_system_prompt.txt") + assert backend.last_result is not None + assert backend.last_result_summary["baseline_pass_rate"] == 0.5 def test_sdk_backend_passes_update_source_true(tmp_path: Path, monkeypatch): @@ -182,12 +185,38 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert report.run["mode"] == "sdk" assert report.run["update_source"] is False assert report.selected_candidate == "sdk_best" - assert report.gate_decisions[0].gate_status == "not_applicable" - assert "SDK optimizer result does not expose per-case validation results" in payload - assert "sdk_best (not_applicable)" in markdown + assert report.baseline_validation.score == 0.5 + assert report.candidates[0]["validation_result"].score == 0.75 + assert report.gate_decisions[0].validation_score_delta == 0.25 + assert report.gate_decisions[0].candidate_cost == 0.123 + assert report.gate_decisions[0].gate_status == "partial_applied" + assert report.gate_decisions[0].not_applied_checks == [ + "per_case_delta", + "protected_regression", + "new_hard_failure", + "max_score_drop_per_case", + ] + assert report.audit["duration_seconds"] == 12.3 + assert report.audit["total_run_cost"] == 0.123 + assert report.audit["cost"]["total"] == 0.123 + assert report.audit["sdk_result_summary"]["status"] == "SUCCEEDED" + assert report.audit["sdk_result_summary"]["baseline_metric_breakdown"] == {"exact_match": 0.5} + assert report.audit["sdk_result_summary"]["best_metric_breakdown"] == {"exact_match": 0.75} + assert report.audit["sdk_result_summary"]["metric_thresholds"] == {"exact_match": 0.7} + assert report.audit["sdk_result_summary"]["total_token_usage"] == { + "prompt": 100, + "completion": 25, + "total": 125, + } + assert report.audit["sdk_result_summary"]["rounds"][0]["validation_pass_rate"] == 0.75 + assert "partial_applied" in payload + assert "sdk_best (partial_applied)" in markdown + assert "not applied checks: per_case_delta" in markdown assert (output_dir / "runs" / "eval_optimize_loop_sdk" / "input_hashes.json").is_file() assert (output_dir / "runs" / "eval_optimize_loop_sdk" / "prompt_diffs" / "sdk_best.diff").is_file() assert calls["update_source"] is False + assert calls["output_dir"].endswith("sdk_optimizer") + assert report.run["sdk_artifact_dir"].endswith("sdk_optimizer") def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp_path: Path, monkeypatch): @@ -219,6 +248,62 @@ def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp assert report.selected_candidate == "sdk_best" +def test_run_pipeline_mode_sdk_rejects_low_aggregate_validation_improvement(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.505, + pass_rate_improvement=0.005, + total_llm_cost=0.123, + ) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.gate_status == "partial_applied" + assert decision.validation_score_delta == 0.005 + assert any("validation improvement" in reason for reason in decision.reasons) + + +def test_run_pipeline_mode_sdk_rejects_cost_over_budget(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.75, + pass_rate_improvement=0.25, + total_llm_cost=2.0, + ) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.gate_status == "partial_applied" + assert decision.total_run_cost == 2.0 + assert any("cost" in reason for reason in decision.reasons) + + def test_run_pipeline_mode_sdk_passes_update_source_true(tmp_path: Path, monkeypatch): calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") @@ -249,7 +334,17 @@ def test_run_pipeline_mode_sdk_missing_call_agent_is_not_fake_fallback(tmp_path: ) -def _install_fake_sdk(monkeypatch, *, best_prompt: str): +def _install_fake_sdk( + monkeypatch, + *, + best_prompt: str, + status: str = "SUCCEEDED", + baseline_pass_rate: float = 0.5, + best_pass_rate: float = 0.75, + pass_rate_improvement: float = 0.25, + total_llm_cost: float = 0.123, + duration_seconds: float = 12.3, +): calls = {} class FakeTargetPrompt: @@ -266,8 +361,27 @@ async def optimize(**kwargs): calls.update(kwargs) return types.SimpleNamespace( best_prompts={"system_prompt": best_prompt}, - status="SUCCEEDED", - best_pass_rate=1.0, + status=status, + baseline_pass_rate=baseline_pass_rate, + best_pass_rate=best_pass_rate, + pass_rate_improvement=pass_rate_improvement, + baseline_metric_breakdown={"exact_match": baseline_pass_rate}, + best_metric_breakdown={"exact_match": best_pass_rate}, + metric_thresholds={"exact_match": 0.7}, + total_llm_cost=total_llm_cost, + total_token_usage={"prompt": 100, "completion": 25, "total": 125}, + duration_seconds=duration_seconds, + total_rounds=1, + rounds=[ + types.SimpleNamespace( + validation_pass_rate=best_pass_rate, + accepted=True, + failed_case_ids=["case_a"], + round_llm_cost=total_llm_cost, + budget_used=3, + budget_total=10, + ) + ], ) fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") From d6c22e991c835f5ca44f00ba3aac00a25774d4ed Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 21:38:35 +0800 Subject: [PATCH 06/34] Decouple SDK wrapper gate config --- .../optimization/eval_optimize_loop/README.md | 39 ++++- .../eval_optimize_loop/eval_loop/backends.py | 61 ++++++- .../eval_optimize_loop/eval_loop/report.py | 14 ++ .../eval_optimize_loop/run_pipeline.py | 107 +++++++++++- .../tests/test_sdk_backend.py | 165 +++++++++++++++++- 5 files changed, 360 insertions(+), 26 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 8c03b074..17990226 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -66,17 +66,44 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ --output-dir /tmp/eval-optimize-loop-sdk ``` +Optional wrapper gate and multi-prompt form: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode sdk \ + --train path/to/sdk_train.evalset.json \ + --val path/to/sdk_val.evalset.json \ + --optimizer-config path/to/sdk_optimizer.json \ + --gate-config path/to/wrapper_gate.json \ + --target-prompt system_prompt=prompts/system.md \ + --target-prompt router_prompt=prompts/router.md \ + --sdk-call-agent your_package.your_module:call_agent \ + --output-dir /tmp/eval-optimize-loop-sdk +``` + `--sdk-call-agent` must point to an async callable compatible with `AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials -needed by that callable. SDK mode never silently falls back to fake mode. When -the SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult` +needed by that callable. SDK mode never silently falls back to fake mode. + +SDK optimizer config and wrapper gate config are intentionally separate. +`--optimizer-config` is passed unchanged to `AgentOptimizer.optimize`, so it must +follow the SDK `OptimizeConfigFile` schema. Put wrapper-only gate settings in +`--gate-config` (for example `{"gate": {"min_val_score_improvement": 0.05, +"max_total_cost": 1.0}}`). If `--gate-config` is omitted, the wrapper uses the +same default aggregate gate values as the fake example. + +Fake mode is the complete per-case closed loop. SDK mode is the real +`AgentOptimizer` / `TargetPrompt` path with an aggregate wrapper gate. When the +SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult` aggregate fields into the JSON/Markdown report: baseline/best pass rate, -pass-rate improvement, metric breakdowns, token usage, duration, LLM cost, and -round summaries. SDK mode applies an aggregate gate with +pass-rate improvement, metric breakdowns, token usage, duration, LLM cost, +all `best_prompts`, and round summaries. SDK mode applies `gate_status: partial_applied`: it checks `status == SUCCEEDED`, validation improvement against `gate.min_val_score_improvement`, and total LLM cost against -`gate.max_total_cost`. Per-case checks that require full validation case scores -are listed in `not_applied_checks` instead of being silently treated as passed. +`gate.max_total_cost`. Protected-case regression, new-hard-failure, per-case +delta, and per-case score-drop checks are not claimed in SDK mode unless the SDK +exposes full per-case validation scores; they are listed in +`not_applied_checks`. ## Source Prompt Writes diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 3dec6e8b..a05d4e54 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -54,6 +54,7 @@ class SDKBackend: prompt_path: str | Path call_agent_path: str | None = None update_source: bool = False + target_prompt_paths: dict[str, str | Path] | None = None last_result: Any | None = None last_result_summary: dict[str, Any] | None = None last_artifact_dir: str | None = None @@ -104,7 +105,10 @@ async def optimize_async( except Exception as exc: # pragma: no cover - depends on optional SDK import health raise ValueError(f"sdk mode could not import AgentOptimizer/TargetPrompt: {exc}") from exc - target_prompt = TargetPrompt().add_path("system_prompt", str(self.prompt_path)) + target_prompt_paths = self._target_prompt_paths() + target_prompt = TargetPrompt() + for name, path in target_prompt_paths.items(): + target_prompt.add_path(name, str(path)) result = await AgentOptimizer.optimize( config_path=str(optimizer_config_path), call_agent=call_agent, @@ -121,20 +125,22 @@ async def optimize_async( self.last_result = result self.last_result_summary = _summarize_sdk_result(result) self.last_artifact_dir = str(output_dir) + best_prompts = dict(getattr(result, "best_prompts", {}) or {}) + baseline_prompts = _read_prompt_bundle(target_prompt_paths) return [ CandidatePrompt( candidate_id="sdk_best", - prompt=best_prompt, + prompt=_render_prompt_bundle(best_prompts), rationale="Best prompt returned by AgentOptimizer.optimize.", - prompt_diff=make_unified_diff( - baseline_prompt, - best_prompt, - before_name="baseline_system_prompt.txt", - after_name="sdk_best/system_prompt.txt", - ), + prompt_diff=_render_prompt_bundle_diff(baseline_prompts, best_prompts), ) ] + def _target_prompt_paths(self) -> dict[str, str | Path]: + if self.target_prompt_paths: + return dict(self.target_prompt_paths) + return {"system_prompt": self.prompt_path} + def _load_call_agent(path: str): if ":" not in path: @@ -171,6 +177,8 @@ def _summarize_sdk_result(result: Any) -> dict[str, Any]: "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})), "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)), "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)), + "baseline_prompts": _safe_jsonable(getattr(result, "baseline_prompts", {})), + "best_prompts": _safe_jsonable(getattr(result, "best_prompts", {})), "rounds": [ { "validation_pass_rate": _safe_jsonable(getattr(round_record, "validation_pass_rate", None)), @@ -197,3 +205,40 @@ def _safe_jsonable(value: Any) -> Any: if isinstance(value, (str, int, float, bool)) or value is None: return value return repr(value) + + +def _read_prompt_bundle(paths: dict[str, str | Path]) -> dict[str, str]: + return { + name: Path(path).read_text(encoding="utf-8") + for name, path in paths.items() + } + + +def _render_prompt_bundle(prompts: dict[str, str]) -> str: + if set(prompts) == {"system_prompt"}: + return prompts["system_prompt"] + sections = [] + for name in sorted(prompts): + sections.append(f"## {name}\n\n{prompts[name]}") + return "\n\n".join(sections) + + +def _render_prompt_bundle_diff(baseline_prompts: dict[str, str], best_prompts: dict[str, str]) -> str: + if set(baseline_prompts) == {"system_prompt"} and set(best_prompts) == {"system_prompt"}: + return make_unified_diff( + baseline_prompts.get("system_prompt", ""), + best_prompts.get("system_prompt", ""), + before_name="baseline_system_prompt.txt", + after_name="sdk_best/system_prompt.txt", + ) + diffs = [] + for name in sorted(set(baseline_prompts) | set(best_prompts)): + diffs.append( + make_unified_diff( + baseline_prompts.get(name, ""), + best_prompts.get(name, ""), + before_name=f"baseline/{name}.txt", + after_name=f"sdk_best/{name}.txt", + ) + ) + return "\n\n".join(diffs) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index 3b264f4a..5a288345 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -127,6 +127,20 @@ def render_markdown(report: OptimizationReport) -> str: "Update source prompt: " + ("yes" if report.run.get("update_source") else "no (default)"), "", + ]) + if report.run.get("mode") == "sdk": + availability = report.audit.get("sdk_result_availability", {}) + lines.extend([ + "SDK mode uses OptimizeResult aggregate validation metrics. " + "Full train scores and full per-case validation deltas are not exposed by the SDK result.", + "", + "SDK availability: " + f"aggregate_validation_result={availability.get('aggregate_validation_result')}, " + f"full_train_eval_result={availability.get('full_train_eval_result')}, " + f"full_per_case_validation_delta={availability.get('full_per_case_validation_delta')}.", + "", + ]) + lines.extend([ "", "## Gate Reasons", "", diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 6a0c96e4..e51957c1 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -53,6 +53,8 @@ def run_pipeline( trace: bool = False, sdk_call_agent: str | None = None, update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, ) -> OptimizationReport: """Run baseline eval, fake optimization, validation gate, and reports.""" @@ -68,10 +70,13 @@ def run_pipeline( optimizer_config_dict = _read_json_object_for_audit(optimizer_config_path) baseline_prompt = load_prompt(prompt_path) sdk_artifact_dir = Path(output_dir) / "sdk_optimizer" + wrapper_gate_config = _load_sdk_gate_config(gate_config_path) + target_prompt_paths = _parse_target_prompt_paths(target_prompts, default_prompt_path=prompt_path) sdk_backend = SDKBackend( prompt_path=prompt_path, call_agent_path=sdk_call_agent, update_source=update_source, + target_prompt_paths=target_prompt_paths, ) candidates = sdk_backend.optimize( baseline_prompt=baseline_prompt, @@ -93,6 +98,9 @@ def run_pipeline( train_case_count=_count_cases(train_path), validation_case_count=_count_cases(val_path), optimizer_config_dict=optimizer_config_dict, + gate_config=wrapper_gate_config, + gate_config_path=gate_config_path, + target_prompt_paths=target_prompt_paths, ) write_reports(report, output_dir) return report @@ -323,6 +331,9 @@ def _build_sdk_report( train_case_count: int | None, validation_case_count: int | None, optimizer_config_dict: dict[str, Any], + gate_config: dict[str, float], + gate_config_path: str | Path | None, + target_prompt_paths: dict[str, str | Path], ) -> OptimizationReport: input_hashes = _input_hashes( train_path=train_path, @@ -340,7 +351,16 @@ def _build_sdk_report( ) total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0) duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0) - gate_config = _sdk_gate_config(optimizer_config_dict) + availability = { + "aggregate_validation_result": True, + "full_train_eval_result": False, + "full_per_case_validation_delta": False, + } + score_explanation = ( + "SDK mode uses OptimizeResult aggregate validation metrics. " + "The full train EvalResult compatibility field is unavailable and keeps score 0.0; " + "full per-case validation deltas are unavailable and listed in not_applied_checks." + ) baseline_train = EvalResult(prompt_id="baseline", split="train", score=0.0, passed=False, cost=0.0, cases=[]) baseline_validation = EvalResult( @@ -380,6 +400,11 @@ def _build_sdk_report( candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() for candidate in candidates } + field_prompt_hashes = _candidate_prompt_hashes_by_field(candidates, sdk_summary) + target_prompt_hashes = { + name: sha256_file(path) + for name, path in target_prompt_paths.items() + } audit = { "seed": None, "duration_seconds": duration_seconds, @@ -393,6 +418,12 @@ def _build_sdk_report( }, "prompt_hash": input_hashes["prompt"], "candidate_prompt_hashes": prompt_hashes, + "candidate_prompt_hashes_by_field": field_prompt_hashes, + "target_prompt_hashes": target_prompt_hashes, + "sdk_result_availability": availability, + "sdk_score_explanation": score_explanation, + "wrapper_gate_config": dict(gate_config), + "wrapper_gate_config_path": str(gate_config_path) if gate_config_path else None, "total_run_cost": total_llm_cost, "cost": { "baseline": 0.0, @@ -417,6 +448,8 @@ def _build_sdk_report( prompt_path=prompt_path, output_dir=output_dir, update_source=update_source, + gate_config_path=gate_config_path, + target_prompt_paths=target_prompt_paths, ), } run = { @@ -429,6 +462,8 @@ def _build_sdk_report( "validation_cases": validation_case_count, "update_source": update_source, "sdk_artifact_dir": audit["sdk_artifact_dir"], + "sdk_availability": availability, + "wrapper_gate_config_path": str(gate_config_path) if gate_config_path else None, "reproducibility_command": audit["reproducibility_command"], "paths": { "train": str(train_path), @@ -436,6 +471,7 @@ def _build_sdk_report( "optimizer": str(optimizer_config_path), "prompt": str(prompt_path), }, + "target_prompts": {name: str(path) for name, path in target_prompt_paths.items()}, "prompt_source": str(prompt_path), } gate_decisions = [ @@ -469,6 +505,8 @@ def _sdk_reproducibility_command( prompt_path: str | Path, output_dir: str | Path, update_source: bool, + gate_config_path: str | Path | None, + target_prompt_paths: dict[str, str | Path], ) -> str: command = ( "python examples/optimization/eval_optimize_loop/run_pipeline.py " @@ -476,6 +514,11 @@ def _sdk_reproducibility_command( f"--optimizer-config {optimizer_config_path} --prompt {prompt_path} " f"--output-dir {output_dir} --sdk-call-agent module:function" ) + for name, path in target_prompt_paths.items(): + if not (name == "system_prompt" and Path(path) == Path(prompt_path) and len(target_prompt_paths) == 1): + command += f" --target-prompt {name}={path}" + if gate_config_path: + command += f" --gate-config {gate_config_path}" if update_source: command += " --update-source" return command @@ -546,19 +589,25 @@ def _sdk_gate_decision( ) -def _sdk_gate_config(optimizer_config_dict: dict[str, Any]) -> dict[str, float]: - gate_payload = optimizer_config_dict.get("gate") +def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, float]: + if gate_config_path is None: + gate_payload: dict[str, Any] = {} + path_text = "--gate-config" + else: + payload = _read_json_object_for_audit(gate_config_path) + gate_payload = payload.get("gate", payload) + path_text = str(gate_config_path) if gate_payload is None: gate_payload = {} if not isinstance(gate_payload, dict): - raise ValueError("optimizer config field 'gate' must be an object when present") + raise ValueError(f"{path_text}: field 'gate' must be an object when present") min_improvement = gate_payload.get("min_val_score_improvement", 0.01) max_cost = gate_payload.get("max_total_cost", 1.0) if not isinstance(min_improvement, (int, float)) or min_improvement < 0: - raise ValueError("optimizer config field 'gate.min_val_score_improvement' must be a non-negative number") + raise ValueError(f"{path_text}: field 'gate.min_val_score_improvement' must be a non-negative number") if not isinstance(max_cost, (int, float)) or max_cost < 0: - raise ValueError("optimizer config field 'gate.max_total_cost' must be a non-negative number") + raise ValueError(f"{path_text}: field 'gate.max_total_cost' must be a non-negative number") return { "min_val_score_improvement": float(min_improvement), "max_total_cost": float(max_cost), @@ -575,6 +624,44 @@ def _summary_float(summary: dict[str, Any], key: str, default: float) -> float: return default +def _parse_target_prompt_paths( + target_prompts: list[str] | None, + *, + default_prompt_path: str | Path, +) -> dict[str, str | Path]: + if not target_prompts: + return {"system_prompt": default_prompt_path} + parsed: dict[str, str | Path] = {} + for item in target_prompts: + if "=" not in item: + raise ValueError("--target-prompt must use name=path format") + name, path = item.split("=", 1) + name = name.strip() + path = path.strip() + if not name or not path: + raise ValueError("--target-prompt must use non-empty name=path values") + if name in parsed: + raise ValueError(f"--target-prompt duplicate field name {name!r}") + parsed[name] = Path(path) + return parsed + + +def _candidate_prompt_hashes_by_field( + candidates: list[CandidatePrompt], + sdk_summary: dict[str, Any], +) -> dict[str, dict[str, str]]: + best_prompts = sdk_summary.get("best_prompts") + if not isinstance(best_prompts, dict): + return {} + return { + candidate.candidate_id: { + str(name): hashlib.sha256(str(prompt).encode("utf-8")).hexdigest() + for name, prompt in best_prompts.items() + } + for candidate in candidates + } + + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--train", default=str(DEFAULT_TRAIN), help="Path to train.evalset.json") @@ -588,6 +675,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--trace", action="store_true", help="Persist fake model/judge trace details per case") parser.add_argument("--sdk-call-agent", help="Async call_agent target for SDK mode, as module:function") parser.add_argument("--update-source", action="store_true", help="Allow SDK optimizer to write back source prompt") + parser.add_argument("--gate-config", help="Wrapper gate config for SDK mode; separate from SDK optimizer config") + parser.add_argument( + "--target-prompt", + action="append", + help="SDK target prompt path in name=path format. May be repeated. Defaults to system_prompt=--prompt.", + ) return parser.parse_args(argv) @@ -639,6 +732,8 @@ def main(argv: list[str] | None = None) -> OptimizationReport: trace=args.trace, sdk_call_agent=args.sdk_call_agent, update_source=args.update_source, + gate_config_path=args.gate_config, + target_prompts=args.target_prompt, ) output_dir = Path(args.output_dir) print(f"Wrote {output_dir / 'optimization_report.json'}") diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 8b0c1edf..31975e90 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -209,9 +209,16 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo "total": 125, } assert report.audit["sdk_result_summary"]["rounds"][0]["validation_pass_rate"] == 0.75 + assert report.audit["sdk_result_availability"] == { + "aggregate_validation_result": True, + "full_train_eval_result": False, + "full_per_case_validation_delta": False, + } + assert "train EvalResult compatibility field is unavailable" in report.audit["sdk_score_explanation"] assert "partial_applied" in payload assert "sdk_best (partial_applied)" in markdown assert "not applied checks: per_case_delta" in markdown + assert "SDK mode uses OptimizeResult aggregate validation metrics" in markdown assert (output_dir / "runs" / "eval_optimize_loop_sdk" / "input_hashes.json").is_file() assert (output_dir / "runs" / "eval_optimize_loop_sdk" / "prompt_diffs" / "sdk_best.diff").is_file() assert calls["update_source"] is False @@ -248,7 +255,10 @@ def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp assert report.selected_candidate == "sdk_best" -def test_run_pipeline_mode_sdk_rejects_low_aggregate_validation_improvement(tmp_path: Path, monkeypatch): +def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_gate( + tmp_path: Path, + monkeypatch, +): _install_fake_sdk( monkeypatch, best_prompt="optimized prompt", @@ -257,12 +267,13 @@ def test_run_pipeline_mode_sdk_rejects_low_aggregate_validation_improvement(tmp_ pass_rate_improvement=0.005, total_llm_cost=0.123, ) + optimizer_path = _write_sdk_optimizer_config(tmp_path) report = run_pipeline( mode="sdk", train_path=DEFAULT_TRAIN, val_path=DEFAULT_VAL, - optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + optimizer_config_path=optimizer_path, prompt_path=DEFAULT_PROMPT, output_dir=tmp_path / "sdk_run", sdk_call_agent="fake_call_agent_module:call_agent", @@ -276,7 +287,39 @@ def test_run_pipeline_mode_sdk_rejects_low_aggregate_validation_improvement(tmp_ assert any("validation improvement" in reason for reason in decision.reasons) -def test_run_pipeline_mode_sdk_rejects_cost_over_budget(tmp_path: Path, monkeypatch): +def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_improvement( + tmp_path: Path, + monkeypatch, +): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.75, + pass_rate_improvement=0.25, + total_llm_cost=0.123, + ) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.3, max_total_cost=1.0) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.validation_score_delta == 0.25 + assert any("validation improvement" in reason for reason in decision.reasons) + + +def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Path, monkeypatch): _install_fake_sdk( monkeypatch, best_prompt="optimized prompt", @@ -285,15 +328,17 @@ def test_run_pipeline_mode_sdk_rejects_cost_over_budget(tmp_path: Path, monkeypa pass_rate_improvement=0.25, total_llm_cost=2.0, ) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.01, max_total_cost=0.05) report = run_pipeline( mode="sdk", train_path=DEFAULT_TRAIN, val_path=DEFAULT_VAL, - optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), prompt_path=DEFAULT_PROMPT, output_dir=tmp_path / "sdk_run", sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, ) decision = report.gate_decisions[0] @@ -304,6 +349,79 @@ def test_run_pipeline_mode_sdk_rejects_cost_over_budget(tmp_path: Path, monkeypa assert any("cost" in reason for reason in decision.reasons) +def test_run_pipeline_mode_sdk_does_not_pass_wrapper_gate_config_to_agent_optimizer( + tmp_path: Path, + monkeypatch, +): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + optimizer_path = _write_sdk_optimizer_config(tmp_path) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.5, max_total_cost=0.05) + + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=optimizer_path, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + assert Path(calls["config_path"]).resolve() == optimizer_path.resolve() + assert "gate" not in json.loads(Path(calls["config_path"]).read_text(encoding="utf-8")) + assert json.loads(gate_path.read_text(encoding="utf-8"))["gate"]["max_total_cost"] == 0.05 + + +def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk( + monkeypatch, + best_prompts={ + "system_prompt": "optimized system", + "router_prompt": "optimized router", + "skill_prompt": "optimized skill", + }, + ) + system_path = tmp_path / "system.txt" + router_path = tmp_path / "router.txt" + skill_path = tmp_path / "skill.txt" + system_path.write_text("baseline system", encoding="utf-8") + router_path.write_text("baseline router", encoding="utf-8") + skill_path.write_text("baseline skill", encoding="utf-8") + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=system_path, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + target_prompts=[ + f"system_prompt={system_path}", + f"router_prompt={router_path}", + f"skill_prompt={skill_path}", + ], + ) + + assert calls["target_prompt"].paths == [ + ("system_prompt", str(system_path)), + ("router_prompt", str(router_path)), + ("skill_prompt", str(skill_path)), + ] + assert report.audit["sdk_result_summary"]["best_prompts"] == { + "system_prompt": "optimized system", + "router_prompt": "optimized router", + "skill_prompt": "optimized skill", + } + assert "router_prompt" in report.candidates[0]["candidate"].prompt_diff + assert set(report.audit["candidate_prompt_hashes_by_field"]["sdk_best"]) == { + "system_prompt", + "router_prompt", + "skill_prompt", + } + + def test_run_pipeline_mode_sdk_passes_update_source_true(tmp_path: Path, monkeypatch): calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") @@ -337,7 +455,8 @@ def test_run_pipeline_mode_sdk_missing_call_agent_is_not_fake_fallback(tmp_path: def _install_fake_sdk( monkeypatch, *, - best_prompt: str, + best_prompt: str | None = None, + best_prompts: dict[str, str] | None = None, status: str = "SUCCEEDED", baseline_pass_rate: float = 0.5, best_pass_rate: float = 0.75, @@ -359,8 +478,11 @@ class FakeAgentOptimizer: @staticmethod async def optimize(**kwargs): calls.update(kwargs) + result_prompts = best_prompts if best_prompts is not None else { + "system_prompt": "optimized prompt" if best_prompt is None else best_prompt + } return types.SimpleNamespace( - best_prompts={"system_prompt": best_prompt}, + best_prompts=result_prompts, status=status, baseline_pass_rate=baseline_pass_rate, best_pass_rate=best_pass_rate, @@ -397,3 +519,34 @@ async def call_agent(query: str) -> str: call_agent_module.call_agent = call_agent monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) return calls + + +def _write_sdk_optimizer_config(tmp_path: Path) -> Path: + path = tmp_path / "sdk_optimizer.json" + path.write_text( + json.dumps({ + "evaluate": {"metrics": []}, + "optimize": {"algorithm": {"name": "gepa_reflective"}}, + }), + encoding="utf-8", + ) + return path + + +def _write_gate_config( + tmp_path: Path, + *, + min_val_score_improvement: float, + max_total_cost: float, +) -> Path: + path = tmp_path / "wrapper_gate.json" + path.write_text( + json.dumps({ + "gate": { + "min_val_score_improvement": min_val_score_improvement, + "max_total_cost": max_total_cost, + } + }), + encoding="utf-8", + ) + return path From 46b077447e735ab97dbfdc53fdff442a9a5835dc Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 21:54:13 +0800 Subject: [PATCH 07/34] Polish SDK prompt audit handling --- .../optimization/eval_optimize_loop/README.md | 19 ++- .../eval_optimize_loop/eval_loop/backends.py | 19 ++- .../eval_optimize_loop/eval_loop/report.py | 7 +- .../eval_optimize_loop/run_pipeline.py | 45 +++++- .../tests/test_sdk_backend.py | 136 +++++++++++++++++- 5 files changed, 210 insertions(+), 16 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 17990226..241b196a 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -78,6 +78,7 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ --target-prompt system_prompt=prompts/system.md \ --target-prompt router_prompt=prompts/router.md \ --sdk-call-agent your_package.your_module:call_agent \ + --run-id local-sdk-smoke \ --output-dir /tmp/eval-optimize-loop-sdk ``` @@ -92,6 +93,11 @@ follow the SDK `OptimizeConfigFile` schema. Put wrapper-only gate settings in "max_total_cost": 1.0}}`). If `--gate-config` is omitted, the wrapper uses the same default aggregate gate values as the fake example. +`--target-prompt name=path` may be repeated. If omitted, SDK mode keeps the old +single-field behavior and registers `system_prompt=--prompt`. A run can optimize +only `router_prompt`, only `skill_prompt`, or any set of named fields as long as +`OptimizeResult.best_prompts` returns every registered field. + Fake mode is the complete per-case closed loop. SDK mode is the real `AgentOptimizer` / `TargetPrompt` path with an aggregate wrapper gate. When the SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult` @@ -105,6 +111,12 @@ delta, and per-case score-drop checks are not claimed in SDK mode unless the SDK exposes full per-case validation scores; they are listed in `not_applied_checks`. +Fake mode uses a deterministic run id (`eval_optimize_loop_seed_`) so the +example outputs are byte-stable. SDK mode is append-only by default: the wrapper +derives `run.run_id` from the SDK result `started_at` when available, otherwise +from the current UTC timestamp. Pass `--run-id` only when a fixed audit path is +useful for tests or local smoke runs. + ## Source Prompt Writes The default is **no source write-back**. The baseline prompt file is not modified @@ -158,8 +170,11 @@ diffs, and the reproducibility command. `report.py` also writes audit artifacts under `output_dir/runs//`: - `config.snapshot.json`; -- `input_hashes.json`; -- `candidate_prompts//system_prompt.txt`; +- `input_hashes.json` with train, validation, optimizer, prompt, + `target_prompts.`, and optional `gate_config` hashes; +- fake mode: `candidate_prompts//system_prompt.txt`; +- SDK mode: `candidate_prompts//.txt` for every + returned `best_prompts` field; - `case_results/_.json`; - `prompt_diffs/.diff`. diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index a05d4e54..454d4ffc 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -119,13 +119,23 @@ async def optimize_async( update_source=self.update_source, verbose=0, ) - best_prompt = getattr(result, "best_prompts", {}).get("system_prompt") - if not best_prompt: - raise ValueError("sdk mode completed but OptimizeResult.best_prompts['system_prompt'] was missing") + best_prompts = dict(getattr(result, "best_prompts", {}) or {}) + if not best_prompts: + raise ValueError("sdk mode completed but OptimizeResult.best_prompts was empty") + missing_fields = [ + name + for name in target_prompt_paths + if not best_prompts.get(name) + ] + if missing_fields: + missing = ", ".join(sorted(missing_fields)) + raise ValueError( + "sdk mode completed but OptimizeResult.best_prompts is missing registered target fields: " + f"{missing}" + ) self.last_result = result self.last_result_summary = _summarize_sdk_result(result) self.last_artifact_dir = str(output_dir) - best_prompts = dict(getattr(result, "best_prompts", {}) or {}) baseline_prompts = _read_prompt_bundle(target_prompt_paths) return [ CandidatePrompt( @@ -176,6 +186,7 @@ def _summarize_sdk_result(result: Any) -> dict[str, Any]: "total_llm_cost": _safe_jsonable(getattr(result, "total_llm_cost", 0.0)), "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})), "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)), + "started_at": _safe_jsonable(getattr(result, "started_at", None)), "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)), "baseline_prompts": _safe_jsonable(getattr(result, "baseline_prompts", {})), "best_prompts": _safe_jsonable(getattr(result, "best_prompts", {})), diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index 5a288345..a61f9060 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -277,7 +277,12 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None candidate: CandidatePrompt = record["candidate"] candidate_dir = prompt_dir / candidate.candidate_id candidate_dir.mkdir(exist_ok=True) - (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8") + best_prompts = report.audit.get("sdk_result_summary", {}).get("best_prompts", {}) + if report.run.get("mode") == "sdk" and isinstance(best_prompts, dict) and best_prompts: + for field_name, prompt_text in best_prompts.items(): + (candidate_dir / f"{field_name}.txt").write_text(str(prompt_text), encoding="utf-8") + else: + (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8") (diffs_dir / f"{candidate.candidate_id}.diff").write_text(candidate.prompt_diff, encoding="utf-8") for split_name in ("train_result", "validation_result"): split_result = record[split_name] diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index e51957c1..b30a61eb 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -7,6 +7,8 @@ import json import sys import tempfile +from datetime import datetime +from datetime import timezone from pathlib import Path from typing import Any @@ -55,6 +57,7 @@ def run_pipeline( update_source: bool = False, gate_config_path: str | Path | None = None, target_prompts: list[str] | None = None, + run_id: str | None = None, ) -> OptimizationReport: """Run baseline eval, fake optimization, validation gate, and reports.""" @@ -101,6 +104,8 @@ def run_pipeline( gate_config=wrapper_gate_config, gate_config_path=gate_config_path, target_prompt_paths=target_prompt_paths, + sdk_call_agent=sdk_call_agent, + run_id=run_id, ) write_reports(report, output_dir) return report @@ -334,6 +339,8 @@ def _build_sdk_report( gate_config: dict[str, float], gate_config_path: str | Path | None, target_prompt_paths: dict[str, str | Path], + sdk_call_agent: str | None, + run_id: str | None, ) -> OptimizationReport: input_hashes = _input_hashes( train_path=train_path, @@ -351,6 +358,14 @@ def _build_sdk_report( ) total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0) duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0) + effective_run_id = run_id or _default_sdk_run_id(sdk_summary) + target_prompt_hashes = { + name: sha256_file(path) + for name, path in target_prompt_paths.items() + } + input_hashes["target_prompts"] = target_prompt_hashes + if gate_config_path: + input_hashes["gate_config"] = sha256_file(gate_config_path) availability = { "aggregate_validation_result": True, "full_train_eval_result": False, @@ -401,10 +416,6 @@ def _build_sdk_report( for candidate in candidates } field_prompt_hashes = _candidate_prompt_hashes_by_field(candidates, sdk_summary) - target_prompt_hashes = { - name: sha256_file(path) - for name, path in target_prompt_paths.items() - } audit = { "seed": None, "duration_seconds": duration_seconds, @@ -450,10 +461,12 @@ def _build_sdk_report( update_source=update_source, gate_config_path=gate_config_path, target_prompt_paths=target_prompt_paths, + sdk_call_agent=sdk_call_agent, + run_id=effective_run_id, ), } run = { - "run_id": "eval_optimize_loop_sdk", + "run_id": effective_run_id, "mode": "sdk", "fake_model": False, "fake_judge": False, @@ -507,18 +520,21 @@ def _sdk_reproducibility_command( update_source: bool, gate_config_path: str | Path | None, target_prompt_paths: dict[str, str | Path], + sdk_call_agent: str | None, + run_id: str, ) -> str: command = ( "python examples/optimization/eval_optimize_loop/run_pipeline.py " f"--mode sdk --train {train_path} --val {val_path} " f"--optimizer-config {optimizer_config_path} --prompt {prompt_path} " - f"--output-dir {output_dir} --sdk-call-agent module:function" + f"--output-dir {output_dir} --sdk-call-agent {sdk_call_agent or ''}" ) for name, path in target_prompt_paths.items(): if not (name == "system_prompt" and Path(path) == Path(prompt_path) and len(target_prompt_paths) == 1): command += f" --target-prompt {name}={path}" if gate_config_path: command += f" --gate-config {gate_config_path}" + command += f" --run-id {run_id}" if update_source: command += " --update-source" return command @@ -624,6 +640,21 @@ def _summary_float(summary: dict[str, Any], key: str, default: float) -> float: return default +def _default_sdk_run_id(sdk_summary: dict[str, Any]) -> str: + started_at = sdk_summary.get("started_at") + if isinstance(started_at, str) and started_at.strip(): + source = started_at.strip() + else: + source = datetime.now(timezone.utc).isoformat(timespec="seconds") + safe = [] + for char in source: + if char.isalnum() or char in {"-", "_"}: + safe.append(char) + else: + safe.append("-") + return "eval_optimize_loop_sdk_" + "".join(safe).strip("-") + + def _parse_target_prompt_paths( target_prompts: list[str] | None, *, @@ -681,6 +712,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: action="append", help="SDK target prompt path in name=path format. May be repeated. Defaults to system_prompt=--prompt.", ) + parser.add_argument("--run-id", help="Optional report/audit run id. Fake mode keeps its deterministic default.") return parser.parse_args(argv) @@ -734,6 +766,7 @@ def main(argv: list[str] | None = None) -> OptimizationReport: update_source=args.update_source, gate_config_path=args.gate_config, target_prompts=args.target_prompt, + run_id=args.run_id, ) output_dir = Path(args.output_dir) print(f"Wrote {output_dir / 'optimization_report.json'}") diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 31975e90..c3caac10 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -55,6 +55,90 @@ def test_sdk_backend_calls_agent_optimizer_and_converts_best_prompt(tmp_path: Pa assert backend.last_result_summary["baseline_pass_rate"] == 0.5 +def test_sdk_backend_default_target_prompt_uses_system_prompt_from_prompt_path(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"system_prompt": "optimized system"}) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline system", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize( + baseline_prompt="baseline system", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] + assert candidates[0].prompt == "optimized system" + + +def test_sdk_backend_router_prompt_only_can_succeed(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"router_prompt": "optimized router"}) + router_path = tmp_path / "router.txt" + router_path.write_text("baseline router", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={"router_prompt": router_path}, + ).optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("router_prompt", str(router_path))] + assert candidates[0].prompt == "## router_prompt\n\noptimized router" + + +def test_sdk_backend_skill_prompt_only_can_succeed(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"skill_prompt": "optimized skill"}) + skill_path = tmp_path / "skill.txt" + skill_path.write_text("baseline skill", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={"skill_prompt": skill_path}, + ).optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("skill_prompt", str(skill_path))] + assert candidates[0].prompt == "## skill_prompt\n\noptimized skill" + + +def test_sdk_backend_missing_registered_best_prompt_field_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompts={"router_prompt": "optimized router"}) + router_path = tmp_path / "router.txt" + skill_path = tmp_path / "skill.txt" + router_path.write_text("baseline router", encoding="utf-8") + skill_path.write_text("baseline skill", encoding="utf-8") + backend = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={"router_prompt": router_path, "skill_prompt": skill_path}, + ) + + with pytest.raises(ValueError, match="best_prompts.*missing registered target fields.*skill_prompt"): + backend.optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + def test_sdk_backend_passes_update_source_true(tmp_path: Path, monkeypatch): calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") prompt_path = tmp_path / "prompt.txt" @@ -123,7 +207,7 @@ def test_sdk_backend_empty_best_prompt_error_is_clear(tmp_path: Path, monkeypatc prompt_path.write_text("baseline", encoding="utf-8") backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") - with pytest.raises(ValueError, match="best_prompts\\['system_prompt'\\]"): + with pytest.raises(ValueError, match="missing registered target fields.*system_prompt"): backend.optimize( baseline_prompt="baseline", train_path=tmp_path / "train.evalset.json", @@ -177,6 +261,7 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo prompt_path=DEFAULT_PROMPT, output_dir=tmp_path / "sdk_run", sdk_call_agent="fake_call_agent_module:call_agent", + run_id="sdk_test_run", ) output_dir = tmp_path / "sdk_run" @@ -219,8 +304,10 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert "sdk_best (partial_applied)" in markdown assert "not applied checks: per_case_delta" in markdown assert "SDK mode uses OptimizeResult aggregate validation metrics" in markdown - assert (output_dir / "runs" / "eval_optimize_loop_sdk" / "input_hashes.json").is_file() - assert (output_dir / "runs" / "eval_optimize_loop_sdk" / "prompt_diffs" / "sdk_best.diff").is_file() + assert "fake_call_agent_module:call_agent" in report.run["reproducibility_command"] + assert "module:function" not in report.run["reproducibility_command"] + assert (output_dir / "runs" / "sdk_test_run" / "input_hashes.json").is_file() + assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / "sdk_best.diff").is_file() assert calls["update_source"] is False assert calls["output_dir"].endswith("sdk_optimizer") assert report.run["sdk_artifact_dir"].endswith("sdk_optimizer") @@ -255,6 +342,27 @@ def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp assert report.selected_candidate == "sdk_best" +def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + started_at="2026-07-04T12:34:56+00:00", + ) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + assert report.run["run_id"] == "eval_optimize_loop_sdk_2026-07-04T12-34-56-00-00" + assert (tmp_path / "sdk_run" / "runs" / report.run["run_id"]).is_dir() + + def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_gate( tmp_path: Path, monkeypatch, @@ -402,6 +510,8 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: f"router_prompt={router_path}", f"skill_prompt={skill_path}", ], + gate_config_path=_write_gate_config(tmp_path, min_val_score_improvement=0.01, max_total_cost=1.0), + run_id="sdk_multi_target", ) assert calls["target_prompt"].paths == [ @@ -420,6 +530,23 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: "router_prompt", "skill_prompt", } + run_dir = tmp_path / "sdk_run" / "runs" / "sdk_multi_target" + assert (run_dir / "candidate_prompts" / "sdk_best" / "system_prompt.txt").read_text( + encoding="utf-8" + ) == "optimized system" + assert (run_dir / "candidate_prompts" / "sdk_best" / "router_prompt.txt").read_text( + encoding="utf-8" + ) == "optimized router" + assert (run_dir / "candidate_prompts" / "sdk_best" / "skill_prompt.txt").read_text( + encoding="utf-8" + ) == "optimized skill" + input_hashes = json.loads((run_dir / "input_hashes.json").read_text(encoding="utf-8")) + assert set(input_hashes["target_prompts"]) == {"system_prompt", "router_prompt", "skill_prompt"} + assert "gate_config" in input_hashes + command = report.run["reproducibility_command"] + assert "--sdk-call-agent fake_call_agent_module:call_agent" in command + assert f"--target-prompt router_prompt={router_path}" in command + assert "--gate-config" in command def test_run_pipeline_mode_sdk_passes_update_source_true(tmp_path: Path, monkeypatch): @@ -438,6 +565,7 @@ def test_run_pipeline_mode_sdk_passes_update_source_true(tmp_path: Path, monkeyp assert report.run["update_source"] is True assert calls["update_source"] is True + assert "--update-source" in report.run["reproducibility_command"] def test_run_pipeline_mode_sdk_missing_call_agent_is_not_fake_fallback(tmp_path: Path): @@ -463,6 +591,7 @@ def _install_fake_sdk( pass_rate_improvement: float = 0.25, total_llm_cost: float = 0.123, duration_seconds: float = 12.3, + started_at: str | None = None, ): calls = {} @@ -493,6 +622,7 @@ async def optimize(**kwargs): total_llm_cost=total_llm_cost, total_token_usage={"prompt": 100, "completion": 25, "total": 125}, duration_seconds=duration_seconds, + started_at=started_at, total_rounds=1, rounds=[ types.SimpleNamespace( From 227827cb6193a0fce243247507ecdd4b3f3b9d7a Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 22:14:56 +0800 Subject: [PATCH 08/34] Harden SDK audit reproducibility --- .../optimization/eval_optimize_loop/README.md | 15 ++-- .../eval_optimize_loop/eval_loop/backends.py | 17 +++-- .../eval_optimize_loop/eval_loop/report.py | 1 + .../eval_optimize_loop/run_pipeline.py | 73 ++++++++++++++----- .../tests/test_sdk_backend.py | 60 ++++++++++++++- 5 files changed, 135 insertions(+), 31 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 241b196a..fa541c8a 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -84,7 +84,10 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ `--sdk-call-agent` must point to an async callable compatible with `AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials -needed by that callable. SDK mode never silently falls back to fake mode. +needed by that callable. SDK mode never silently falls back to fake mode. The +generated reproducibility command records the actual `--sdk-call-agent` +`module:function` target and file/config paths, but it does not record API keys +or other provider secrets. SDK optimizer config and wrapper gate config are intentionally separate. `--optimizer-config` is passed unchanged to `AgentOptimizer.optimize`, so it must @@ -113,9 +116,10 @@ exposes full per-case validation scores; they are listed in Fake mode uses a deterministic run id (`eval_optimize_loop_seed_`) so the example outputs are byte-stable. SDK mode is append-only by default: the wrapper -derives `run.run_id` from the SDK result `started_at` when available, otherwise -from the current UTC timestamp. Pass `--run-id` only when a fixed audit path is -useful for tests or local smoke runs. +derives a compact UTC `run.run_id` from the SDK result `started_at` when +available, otherwise from the current UTC timestamp. Pass `--run-id` only when +a fixed audit path is useful for tests or local smoke runs; only explicit +`--run-id` values are included in the reproducibility command. ## Source Prompt Writes @@ -174,7 +178,8 @@ diffs, and the reproducibility command. `target_prompts.`, and optional `gate_config` hashes; - fake mode: `candidate_prompts//system_prompt.txt`; - SDK mode: `candidate_prompts//.txt` for every - returned `best_prompts` field; + returned `best_prompts` field, plus `bundle.txt` with the combined prompt + shown in the wrapper report; - `case_results/_.json`; - `prompt_diffs/.diff`. diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 454d4ffc..95a97bb5 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -122,17 +122,24 @@ async def optimize_async( best_prompts = dict(getattr(result, "best_prompts", {}) or {}) if not best_prompts: raise ValueError("sdk mode completed but OptimizeResult.best_prompts was empty") - missing_fields = [ - name - for name in target_prompt_paths - if not best_prompts.get(name) - ] + missing_fields = [name for name in target_prompt_paths if name not in best_prompts] if missing_fields: missing = ", ".join(sorted(missing_fields)) raise ValueError( "sdk mode completed but OptimizeResult.best_prompts is missing registered target fields: " f"{missing}" ) + empty_fields = [ + name + for name in target_prompt_paths + if not isinstance(best_prompts[name], str) or not best_prompts[name].strip() + ] + if empty_fields: + empty = ", ".join(sorted(empty_fields)) + raise ValueError( + "sdk mode completed but OptimizeResult.best_prompts contained empty registered target fields: " + f"{empty}" + ) self.last_result = result self.last_result_summary = _summarize_sdk_result(result) self.last_artifact_dir = str(output_dir) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index a61f9060..2c475c88 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -281,6 +281,7 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None if report.run.get("mode") == "sdk" and isinstance(best_prompts, dict) and best_prompts: for field_name, prompt_text in best_prompts.items(): (candidate_dir / f"{field_name}.txt").write_text(str(prompt_text), encoding="utf-8") + (candidate_dir / "bundle.txt").write_text(candidate.prompt, encoding="utf-8") else: (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8") (diffs_dir / f"{candidate.candidate_id}.diff").write_text(candidate.prompt_diff, encoding="utf-8") diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index b30a61eb..ee44609d 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -5,6 +5,8 @@ import argparse import hashlib import json +import math +import shlex import sys import tempfile from datetime import datetime @@ -462,7 +464,7 @@ def _build_sdk_report( gate_config_path=gate_config_path, target_prompt_paths=target_prompt_paths, sdk_call_agent=sdk_call_agent, - run_id=effective_run_id, + run_id=run_id, ), } run = { @@ -521,23 +523,36 @@ def _sdk_reproducibility_command( gate_config_path: str | Path | None, target_prompt_paths: dict[str, str | Path], sdk_call_agent: str | None, - run_id: str, + run_id: str | None, ) -> str: - command = ( - "python examples/optimization/eval_optimize_loop/run_pipeline.py " - f"--mode sdk --train {train_path} --val {val_path} " - f"--optimizer-config {optimizer_config_path} --prompt {prompt_path} " - f"--output-dir {output_dir} --sdk-call-agent {sdk_call_agent or ''}" - ) + parts = [ + "python", + "examples/optimization/eval_optimize_loop/run_pipeline.py", + "--mode", + "sdk", + "--train", + str(train_path), + "--val", + str(val_path), + "--optimizer-config", + str(optimizer_config_path), + "--prompt", + str(prompt_path), + "--output-dir", + str(output_dir), + "--sdk-call-agent", + sdk_call_agent or "", + ] for name, path in target_prompt_paths.items(): if not (name == "system_prompt" and Path(path) == Path(prompt_path) and len(target_prompt_paths) == 1): - command += f" --target-prompt {name}={path}" + parts.extend(["--target-prompt", f"{name}={path}"]) if gate_config_path: - command += f" --gate-config {gate_config_path}" - command += f" --run-id {run_id}" + parts.extend(["--gate-config", str(gate_config_path)]) + if run_id: + parts.extend(["--run-id", run_id]) if update_source: - command += " --update-source" - return command + parts.append("--update-source") + return " ".join(shlex.quote(part) for part in parts) def _sdk_gate_decision( @@ -620,16 +635,30 @@ def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, floa min_improvement = gate_payload.get("min_val_score_improvement", 0.01) max_cost = gate_payload.get("max_total_cost", 1.0) - if not isinstance(min_improvement, (int, float)) or min_improvement < 0: - raise ValueError(f"{path_text}: field 'gate.min_val_score_improvement' must be a non-negative number") - if not isinstance(max_cost, (int, float)) or max_cost < 0: - raise ValueError(f"{path_text}: field 'gate.max_total_cost' must be a non-negative number") + if not _is_non_negative_finite_number(min_improvement): + raise ValueError( + f"--gate-config {path_text}: field 'gate.min_val_score_improvement' " + "must be a non-negative finite number" + ) + if not _is_non_negative_finite_number(max_cost): + raise ValueError( + f"--gate-config {path_text}: field 'gate.max_total_cost' must be a non-negative finite number" + ) return { "min_val_score_improvement": float(min_improvement), "max_total_cost": float(max_cost), } +def _is_non_negative_finite_number(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + and float(value) >= 0 + ) + + def _summary_float(summary: dict[str, Any], key: str, default: float) -> float: value = summary.get(key, default) if value is None: @@ -644,8 +673,16 @@ def _default_sdk_run_id(sdk_summary: dict[str, Any]) -> str: started_at = sdk_summary.get("started_at") if isinstance(started_at, str) and started_at.strip(): source = started_at.strip() + try: + normalized = source[:-1] + "+00:00" if source.endswith("Z") else source + parsed = datetime.fromisoformat(normalized) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return "eval_optimize_loop_sdk_" + parsed.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + except ValueError: + pass else: - source = datetime.now(timezone.utc).isoformat(timespec="seconds") + return "eval_optimize_loop_sdk_" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") safe = [] for char in source: if char.isalnum() or char in {"-", "_"}: diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index c3caac10..9606f300 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -139,6 +139,22 @@ def test_sdk_backend_missing_registered_best_prompt_field_is_clear(tmp_path: Pat ) +def test_sdk_backend_empty_best_prompts_dict_error_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompts={}) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="best_prompts was empty"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + def test_sdk_backend_passes_update_source_true(tmp_path: Path, monkeypatch): calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") prompt_path = tmp_path / "prompt.txt" @@ -207,7 +223,7 @@ def test_sdk_backend_empty_best_prompt_error_is_clear(tmp_path: Path, monkeypatc prompt_path.write_text("baseline", encoding="utf-8") backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") - with pytest.raises(ValueError, match="missing registered target fields.*system_prompt"): + with pytest.raises(ValueError, match="contained empty registered target fields.*system_prompt"): backend.optimize( baseline_prompt="baseline", train_path=tmp_path / "train.evalset.json", @@ -359,8 +375,9 @@ def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path sdk_call_agent="fake_call_agent_module:call_agent", ) - assert report.run["run_id"] == "eval_optimize_loop_sdk_2026-07-04T12-34-56-00-00" + assert report.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z" assert (tmp_path / "sdk_run" / "runs" / report.run["run_id"]).is_dir() + assert "--run-id" not in report.run["reproducibility_command"] def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_gate( @@ -457,6 +474,39 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Pa assert any("cost" in reason for reason in decision.reasons) +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + ("min_val_score_improvement", True), + ("max_total_cost", float("nan")), + ("max_total_cost", float("inf")), + ], +) +def test_run_pipeline_mode_sdk_rejects_invalid_gate_numbers( + tmp_path: Path, + monkeypatch, + field_name, + field_value, +): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + gate = {"min_val_score_improvement": 0.01, "max_total_cost": 1.0} + gate[field_name] = field_value + gate_path = tmp_path / "bad_gate.json" + gate_path.write_text(json.dumps({"gate": gate}), encoding="utf-8") + + with pytest.raises(ValueError, match=f"--gate-config.*{field_name}"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + def test_run_pipeline_mode_sdk_does_not_pass_wrapper_gate_config_to_agent_optimizer( tmp_path: Path, monkeypatch, @@ -540,12 +590,16 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: assert (run_dir / "candidate_prompts" / "sdk_best" / "skill_prompt.txt").read_text( encoding="utf-8" ) == "optimized skill" + assert (run_dir / "candidate_prompts" / "sdk_best" / "bundle.txt").read_text( + encoding="utf-8" + ) == report.candidates[0]["candidate"].prompt input_hashes = json.loads((run_dir / "input_hashes.json").read_text(encoding="utf-8")) assert set(input_hashes["target_prompts"]) == {"system_prompt", "router_prompt", "skill_prompt"} assert "gate_config" in input_hashes command = report.run["reproducibility_command"] assert "--sdk-call-agent fake_call_agent_module:call_agent" in command - assert f"--target-prompt router_prompt={router_path}" in command + assert "--target-prompt" in command + assert f"router_prompt={router_path}" in command assert "--gate-config" in command From 270f9a44820f13c29de502eda6de24c75ff63a8d Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 22:28:22 +0800 Subject: [PATCH 09/34] Harden eval optimize audit inputs --- .../eval_optimize_loop/eval_loop/backends.py | 28 +++- .../eval_optimize_loop/eval_loop/report.py | 28 +++- .../eval_optimize_loop/run_pipeline.py | 60 ++++++- .../tests/test_pipeline_fake_mode.py | 23 +++ .../tests/test_sdk_backend.py | 156 ++++++++++++++++++ 5 files changed, 274 insertions(+), 21 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 95a97bb5..38709137 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -4,6 +4,7 @@ import asyncio import importlib +import math from dataclasses import dataclass from pathlib import Path from typing import Any @@ -170,6 +171,8 @@ def _load_call_agent(path: str): call_agent = getattr(module, function_name, None) if call_agent is None: raise ValueError(f"--sdk-call-agent target {path!r} was not found") + if not callable(call_agent): + raise ValueError(f"--sdk-call-agent target {path!r} was found but is not callable") return call_agent @@ -184,13 +187,17 @@ def _has_running_loop() -> bool: def _summarize_sdk_result(result: Any) -> dict[str, Any]: return { "status": _safe_jsonable(getattr(result, "status", None)), - "baseline_pass_rate": _safe_jsonable(getattr(result, "baseline_pass_rate", None)), - "best_pass_rate": _safe_jsonable(getattr(result, "best_pass_rate", None)), - "pass_rate_improvement": _safe_jsonable(getattr(result, "pass_rate_improvement", None)), + "baseline_pass_rate": _safe_result_field( + "baseline_pass_rate", getattr(result, "baseline_pass_rate", None) + ), + "best_pass_rate": _safe_result_field("best_pass_rate", getattr(result, "best_pass_rate", None)), + "pass_rate_improvement": _safe_result_field( + "pass_rate_improvement", getattr(result, "pass_rate_improvement", None) + ), "baseline_metric_breakdown": _safe_jsonable(getattr(result, "baseline_metric_breakdown", {})), "best_metric_breakdown": _safe_jsonable(getattr(result, "best_metric_breakdown", {})), "metric_thresholds": _safe_jsonable(getattr(result, "metric_thresholds", {})), - "total_llm_cost": _safe_jsonable(getattr(result, "total_llm_cost", 0.0)), + "total_llm_cost": _safe_result_field("total_llm_cost", getattr(result, "total_llm_cost", 0.0)), "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})), "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)), "started_at": _safe_jsonable(getattr(result, "started_at", None)), @@ -211,6 +218,13 @@ def _summarize_sdk_result(result: Any) -> dict[str, Any]: } +def _safe_result_field(field_name: str, value: Any) -> Any: + try: + return _safe_jsonable(value) + except ValueError as exc: + raise ValueError(f"SDK OptimizeResult field {field_name} must be a finite number") from exc + + def _safe_jsonable(value: Any) -> Any: if hasattr(value, "model_dump"): return _safe_jsonable(value.model_dump(mode="json")) @@ -220,7 +234,11 @@ def _safe_jsonable(value: Any) -> Any: return {str(key): _safe_jsonable(item) for key, item in value.items()} if isinstance(value, (list, tuple)): return [_safe_jsonable(item) for item in value] - if isinstance(value, (str, int, float, bool)) or value is None: + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("value must be a finite number") + return value + if isinstance(value, (str, int, bool)) or value is None: return value return repr(value) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index 2c475c88..de1a1b35 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re import shutil from pathlib import Path from typing import Any @@ -25,6 +26,7 @@ "--output-dir /tmp/eval-optimize-loop " "--fake-model --fake-judge --trace" ) +ARTIFACT_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") def compute_case_deltas( @@ -106,7 +108,7 @@ def write_reports(report: OptimizationReport, output_dir: str | Path) -> tuple[P def report_to_json(report: OptimizationReport) -> str: - return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True) + "\n" + return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True, allow_nan=False) + "\n" def render_markdown(report: OptimizationReport) -> str: @@ -248,7 +250,7 @@ def render_markdown(report: OptimizationReport) -> str: def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None: - run_id = str(report.run.get("run_id") or "run") + run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) run_dir = output_path / "runs" / run_id if run_dir.exists() and report.run.get("mode") == "fake": shutil.rmtree(run_dir) @@ -275,20 +277,26 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None for record in report.candidates: candidate: CandidatePrompt = record["candidate"] - candidate_dir = prompt_dir / candidate.candidate_id + candidate_name = _safe_artifact_name(candidate.candidate_id) + candidate_dir = prompt_dir / candidate_name candidate_dir.mkdir(exist_ok=True) best_prompts = report.audit.get("sdk_result_summary", {}).get("best_prompts", {}) if report.run.get("mode") == "sdk" and isinstance(best_prompts, dict) and best_prompts: for field_name, prompt_text in best_prompts.items(): - (candidate_dir / f"{field_name}.txt").write_text(str(prompt_text), encoding="utf-8") + field_artifact = _safe_artifact_name(str(field_name)) + (candidate_dir / f"{field_artifact}.txt").write_text(str(prompt_text), encoding="utf-8") (candidate_dir / "bundle.txt").write_text(candidate.prompt, encoding="utf-8") else: (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8") - (diffs_dir / f"{candidate.candidate_id}.diff").write_text(candidate.prompt_diff, encoding="utf-8") + (diffs_dir / f"{candidate_name}.diff").write_text(candidate.prompt_diff, encoding="utf-8") for split_name in ("train_result", "validation_result"): split_result = record[split_name] - path = results_dir / f"{candidate.candidate_id}_{split_result.split}.json" - path.write_text(json.dumps(to_jsonable(split_result), indent=2, sort_keys=True) + "\n", encoding="utf-8") + split_artifact = _safe_artifact_name(str(split_result.split)) + path = results_dir / f"{candidate_name}_{split_artifact}.json" + path.write_text( + json.dumps(to_jsonable(split_result), indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float) -> str: @@ -301,3 +309,9 @@ def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float) if delta < 0: return "score_down" return "unchanged" + + +def _safe_artifact_name(name: str) -> str: + if name in {"", ".", ".."} or not ARTIFACT_NAME_RE.fullmatch(name): + raise ValueError(f"unsafe audit artifact name: {name!r}") + return name diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index ee44609d..9d83ef86 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -6,6 +6,7 @@ import hashlib import json import math +import re import shlex import sys import tempfile @@ -42,6 +43,8 @@ DEFAULT_OPTIMIZER_CONFIG = HERE / "data" / "optimizer.json" DEFAULT_PROMPT = HERE / "prompts" / "baseline_system_prompt.txt" DEFAULT_OUTPUT_DIR = Path(tempfile.gettempdir()) / "eval-optimize-loop" +TARGET_PROMPT_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +RUN_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") def run_pipeline( @@ -65,6 +68,8 @@ def run_pipeline( if mode not in {"fake", "sdk"}: raise ValueError("field 'mode' must be one of: fake, sdk") + if run_id is not None: + run_id = validate_run_id(run_id) if mode == "fake" and (not fake_model or not fake_judge): raise ValueError( "fake mode requires fake_model=True and fake_judge=True. Pass --fake-model --fake-judge " @@ -109,6 +114,8 @@ def run_pipeline( sdk_call_agent=sdk_call_agent, run_id=run_id, ) + if run_id is None: + _resolve_default_sdk_run_id_collision(report, output_dir) write_reports(report, output_dir) return report @@ -351,14 +358,15 @@ def _build_sdk_report( prompt_path=prompt_path, ) sdk_summary = sdk_backend.last_result_summary or {} - baseline_pass_rate = _summary_float(sdk_summary, "baseline_pass_rate", 0.0) - best_pass_rate = _summary_float(sdk_summary, "best_pass_rate", baseline_pass_rate) + baseline_pass_rate = _summary_float(sdk_summary, "baseline_pass_rate", 0.0, required=True) + best_pass_rate = _summary_float(sdk_summary, "best_pass_rate", baseline_pass_rate, required=True) pass_rate_improvement = _summary_float( sdk_summary, "pass_rate_improvement", best_pass_rate - baseline_pass_rate, + required=True, ) - total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0) + total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0) effective_run_id = run_id or _default_sdk_run_id(sdk_summary) target_prompt_hashes = { @@ -562,8 +570,8 @@ def _sdk_gate_decision( gate_config: dict[str, float], ) -> GateDecision: status = str(sdk_summary.get("status") or "UNKNOWN") - improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0) - total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0) + improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0, required=True) + total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) min_improvement = gate_config["min_val_score_improvement"] max_cost = gate_config["max_total_cost"] @@ -659,14 +667,25 @@ def _is_non_negative_finite_number(value: Any) -> bool: ) -def _summary_float(summary: dict[str, Any], key: str, default: float) -> float: +def _summary_float(summary: dict[str, Any], key: str, default: float, *, required: bool = False) -> float: value = summary.get(key, default) if value is None: return default + if isinstance(value, bool): + if required: + raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") + return default try: - return float(value) + parsed = float(value) except (TypeError, ValueError): + if required: + raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") + return default + if not math.isfinite(parsed): + if required: + raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") return default + return parsed def _default_sdk_run_id(sdk_summary: dict[str, Any]) -> str: @@ -704,9 +723,12 @@ def _parse_target_prompt_paths( if "=" not in item: raise ValueError("--target-prompt must use name=path format") name, path = item.split("=", 1) - name = name.strip() path = path.strip() - if not name or not path: + if not TARGET_PROMPT_FIELD_RE.fullmatch(name): + raise ValueError( + f"--target-prompt field name {name!r} is invalid; use /^[A-Za-z_][A-Za-z0-9_]*$/" + ) + if not path: raise ValueError("--target-prompt must use non-empty name=path values") if name in parsed: raise ValueError(f"--target-prompt duplicate field name {name!r}") @@ -714,6 +736,26 @@ def _parse_target_prompt_paths( return parsed +def validate_run_id(run_id: str) -> str: + if not isinstance(run_id, str): + raise ValueError(f"--run-id value {run_id!r} must be a string") + if run_id in {"", ".", ".."} or not RUN_ID_RE.fullmatch(run_id): + raise ValueError(f"--run-id value {run_id!r} is invalid") + return run_id + + +def _resolve_default_sdk_run_id_collision(report: OptimizationReport, output_dir: str | Path) -> None: + base_run_id = str(report.run.get("run_id") or "") + validate_run_id(base_run_id) + run_root = Path(output_dir) / "runs" + candidate = base_run_id + suffix = 1 + while (run_root / candidate).exists(): + candidate = f"{base_run_id}-{suffix}" + suffix += 1 + report.run["run_id"] = candidate + + def _candidate_prompt_hashes_by_field( candidates: list[CandidatePrompt], sdk_summary: dict[str, Any], diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py index d43810f8..85247d7e 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -10,6 +10,7 @@ from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline +from examples.optimization.eval_optimize_loop.eval_loop.report import report_to_json def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): @@ -69,6 +70,7 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): "total_run_cost", } assert payload["run"]["reproducibility_command"].startswith("python examples/optimization") + assert payload["run"]["run_id"] == "eval_optimize_loop_seed_91" assert payload["audit"]["total_run_cost"] == payload["audit"]["cost"]["total"] assert "candidate_001_overfit" in payload["audit"]["prompt_diffs"] assert "candidate_002_safe" in payload["audit"]["prompt_diffs"] @@ -157,3 +159,24 @@ def test_cli_mode_fake_and_legacy_fake_flags_both_run(tmp_path: Path): assert (first / "optimization_report.json").is_file() assert (second / "optimization_report.md").is_file() + + +def test_report_to_json_rejects_nan_values(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + report.audit["bad_float"] = float("nan") + + try: + report_to_json(report) + except ValueError as exc: + assert "Out of range float values are not JSON compliant" in str(exc) + else: + raise AssertionError("report_to_json should reject NaN") + + +def test_fake_report_json_remains_strict_json(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + payload = report_to_json(report) + + assert "NaN" not in payload + assert "Infinity" not in payload + assert json.loads(payload)["selected_candidate"] == "candidate_002_safe" diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 9606f300..de071c97 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -188,6 +188,22 @@ def test_sdk_backend_call_agent_import_failure_names_target(tmp_path: Path): ) +def test_sdk_backend_call_agent_must_be_callable(tmp_path: Path, monkeypatch): + call_agent_module = types.ModuleType("fake_call_agent_module") + call_agent_module.call_agent = "not callable" + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="--sdk-call-agent.*fake_call_agent_module:call_agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + def test_sdk_backend_sdk_import_failure_is_clear(tmp_path: Path, monkeypatch): call_agent_module = types.ModuleType("fake_call_agent_module") @@ -380,6 +396,66 @@ def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path assert "--run-id" not in report.run["reproducibility_command"] +def test_run_pipeline_mode_sdk_default_run_id_collision_gets_suffix(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + started_at="2026-07-04T12:34:56+00:00", + ) + + first = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + second = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + assert first.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z" + assert second.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z-1" + assert (tmp_path / "sdk_run" / "runs" / first.run["run_id"]).is_dir() + assert (tmp_path / "sdk_run" / "runs" / second.run["run_id"]).is_dir() + + +def test_run_pipeline_mode_sdk_explicit_run_id_stays_stable(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + first = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id="valid_20260704-1.ok", + ) + second = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id="valid_20260704-1.ok", + ) + + assert first.run["run_id"] == "valid_20260704-1.ok" + assert second.run["run_id"] == "valid_20260704-1.ok" + + def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_gate( tmp_path: Path, monkeypatch, @@ -507,6 +583,86 @@ def test_run_pipeline_mode_sdk_rejects_invalid_gate_numbers( ) +@pytest.mark.parametrize("run_id", ["../../escape", "a/b", "", ".", "..", "has space", "a\\b"]) +def test_run_pipeline_rejects_invalid_run_id(tmp_path: Path, monkeypatch, run_id): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + with pytest.raises(ValueError, match="--run-id") as exc_info: + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id=run_id, + ) + assert repr(run_id) in str(exc_info.value) + + +@pytest.mark.parametrize( + "field_name", + ["../router", "router/prompt", "router prompt", "router.prompt", "router-prompt", "", " router_prompt"], +) +def test_run_pipeline_mode_sdk_rejects_invalid_target_prompt_field_names( + tmp_path: Path, + monkeypatch, + field_name, +): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="--target-prompt") as exc_info: + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + target_prompts=[f"{field_name}={prompt_path}"], + ) + assert repr(field_name) in str(exc_info.value) + + +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + ("pass_rate_improvement", float("nan")), + ("total_llm_cost", float("inf")), + ("best_pass_rate", "bad"), + ], +) +def test_run_pipeline_mode_sdk_rejects_non_finite_or_bad_numeric_summary( + tmp_path: Path, + monkeypatch, + field_name, + field_value, +): + kwargs = { + "baseline_pass_rate": 0.5, + "best_pass_rate": 0.75, + "pass_rate_improvement": 0.25, + "total_llm_cost": 0.123, + } + kwargs[field_name] = field_value + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt", **kwargs) + + with pytest.raises(ValueError, match=f"SDK OptimizeResult field {field_name} must be a finite number"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + def test_run_pipeline_mode_sdk_does_not_pass_wrapper_gate_config_to_agent_optimizer( tmp_path: Path, monkeypatch, From 20e1957235d0efa74aae4ec1780342f383e93a6d Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 10:16:46 +0800 Subject: [PATCH 10/34] docs: design issue 91 safe optimization loop --- ...-07-10-issue-91-safe-closed-loop-design.md | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-10-issue-91-safe-closed-loop-design.md diff --git a/docs/superpowers/specs/2026-07-10-issue-91-safe-closed-loop-design.md b/docs/superpowers/specs/2026-07-10-issue-91-safe-closed-loop-design.md new file mode 100644 index 00000000..1d2a89d0 --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-issue-91-safe-closed-loop-design.md @@ -0,0 +1,245 @@ +# Issue #91 安全评测优化闭环设计 + +## 背景 + +PR #119 已经提供 fake 评测、候选 prompt、gate 和 JSON/Markdown 报告，但已发布提交中没有一条路径同时完成真实 baseline 评测、候选逐 case 复评、完整 gate 和 gate 后回写。fake model 还会读取 expectation、split、protected 等评测真值，因此现有测试不能证明隐藏样本决策准确率。SDK 模式把 `--update-source` 直接传给 `AgentOptimizer`，候选可能在 wrapper gate 拒绝前写入源 prompt。 + +本设计的边界是通过 Issue #91 的验收并安全合并。与验收无关的通用框架重构不纳入本次修改。 + +## 目标 + +1. fake 和 SDK 共用同一条“评测—归因—优化—复评—gate—审计—可选回写”流程。 +2. SDK 使用真实 `AgentEvaluator`、`AgentOptimizer` 和 `TargetPrompt`，但外部模型调用仍可在 CI 中替换为确定性实现。 +3. gate 只消费完整、可核验的逐 case 结果；不再允许 `cases=[]` 或 `partial_applied` 的候选被选中。 +4. 源 prompt 只在候选通过完整 gate 且用户显式指定 `--update-source` 后写回。 +5. 提供可执行证据证明隐藏决策准确率、失败归因准确率、过拟合拒绝和三分钟性能要求。 +6. 报告、输入哈希、耗时、成本和运行产物可以复核，不泄露贡献者机器的绝对路径。 + +## 非目标 + +- 不修改 `AgentEvaluator`、`AgentOptimizer` 或 GEPA 的公共 API。 +- 不引入新的远程服务、数据库或长期任务系统。 +- 不要求 CI 使用真实 API key。 +- 不把示例扩展成通用生产部署平台。 + +## 选定方案 + +采用统一闭环编排器。fake 与 SDK 仅负责各自的 evaluate/optimize 适配，候选选择、gate、审计和写回语义由共享 pipeline 实现。 + +分别修补两条路径虽然改动较少，但会保留两套不同的 gate 和报告语义；完全 SDK-first 会放大重构范围。统一编排器在验收覆盖、维护成本和合并风险之间更平衡，并可直接承接当前工作树中已有的 SDK EvalSet 与 AgentEvaluator 适配工作。 + +## 组件与文件职责 + +### `run_pipeline.py` + +保留 CLI、路径解析和同步兼容入口。新增异步核心入口的薄包装：同步 `run_pipeline()` 在没有活动事件循环时调用 `asyncio.run(run_pipeline_async(...))`；异步调用者直接使用 `run_pipeline_async()`。该文件不再实现候选循环、SDK 报告拼装或写回逻辑。 + +### `eval_loop/pipeline.py` + +新增共享编排器，负责： + +1. 冻结输入与 prompt 快照； +2. baseline train/validation 评测； +3. 失败归因； +4. 调用 optimizer backend； +5. 对每个候选重新执行 train/validation 评测； +6. 计算逐 case delta 和完整 gate； +7. 选择候选； +8. 写审计产物； +9. 在允许时执行安全回写。 + +### `eval_loop/backends.py` + +backend 统一暴露异步接口： + +```python +class EvaluationBackend(Protocol): + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: Path, + split: str, + trace: bool, + artifact_dir: Path, + ) -> EvalResult: ... + + +class OptimizationBackend(Protocol): + async def optimize( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: Path, + validation_path: Path, + config_path: Path, + artifact_dir: Path, + ) -> OptimizationResult: ... +``` + +`SDKBackend.optimize()` 始终调用 `AgentOptimizer.optimize(update_source=False)`。它从 `OptimizeResult.rounds[].candidate_prompts` 和 `best_prompts` 提取、去重候选；每个候选由共享 pipeline 做完整 train/validation 复评。`SDKBackend.evaluate()` 使用真实 `AgentEvaluator`，并把每个 case 的逐 metric 分数、pass/fail、原因、证据和可用 trace 转为统一 schema。关键轨迹从 `EvalCaseResult.eval_metric_result_per_invocation[].actual_invocation` 提取用户输入、工具调用和最终回复；SDK 没有提供某类轨迹时显式设置 `trace_available=False`，不能伪造空轨迹为“已采集”。 + +`FakeBackend.optimize()` 只接收 baseline train 结果、失败归因和 optimizer 配置，不接收 validation 评测结果。它根据观察到的失败类别生成确定性候选，而不是无条件返回固定候选。 + +### `eval_loop/gate.py` + +保留唯一 gate 实现。gate 检查： + +- validation 总分提升阈值； +- 训练提升但 validation 不提升的过拟合； +- 新 hard fail； +- protected case 退化； +- 单 case 最大降分； +- 可选成本预算； +- baseline 与 candidate case ID 集合一致。 + +候选缺失逐 case 数据、成本 gate 所需数据不完整或评测失败时，gate 必须拒绝并返回明确原因。 + +### `eval_loop/writeback.py` + +新增 prompt 快照和回写组件： + +- 保存每个 prompt 文件的原始字节与 SHA-256； +- 候选复评期间临时应用 prompt，并在 `finally` 中恢复； +- 恢复后校验哈希，恢复失败立即终止 pipeline； +- 最终回写前执行 compare-and-swap 检查，避免覆盖并发修改； +- 多 prompt 使用临时文件和 `os.replace`，任一失败时回滚所有已写字段； +- 返回结构化 `WritebackResult`。 + +### `eval_loop/report.py` + +只负责 schema 序列化、Markdown 渲染和 run-specific 审计目录。它不再推导 gate 或 SDK 特殊结果。 + +## 统一数据模型 + +`CaseResult` 增加逐 metric 数据和 trace 可用性： + +```python +@dataclass(frozen=True) +class CaseResult: + case_id: str + split: str + score: float + metrics: dict[str, float] + passed: bool + output: str + trace: dict[str, object] + trace_available: bool + failure_category: str | None + failure_reason: str | None + evidence: str | None + cost: float + hard_failed: bool + expected_failure_category: str | None +``` + +新增以下审计模型： + +- `OptimizationRound`：round ID、候选 prompt bundle、修改理由、optimizer 指标、成本和耗时； +- `CostSummary`：optimizer、evaluator、agent、total 和 `complete`； +- `WritebackResult`：`rejected` 表示没有候选通过 gate，`not_requested` 表示候选通过但用户未要求回写，`applied` 表示回写成功，`rolled_back` 表示写入失败且已完整恢复，`rollback_failed` 表示恢复不完整；同时记录前后哈希和错误原因； +- `OptimizationResult`：所有去重候选、round 记录和 backend 原始摘要。 + +## 端到端数据流 + +1. 解析并严格验证四类输入文件、gate 配置和 target prompt 路径。 +2. 生成唯一 `run_id`，创建 `runs/.tmp/`，启动 `perf_counter`。 +3. 对输入文件和源 prompt 做字节级快照与 SHA-256。 +4. 用 baseline prompt 分别评测 train 和 validation；任一 baseline 评测缺 case 或恢复 prompt 失败时终止。 +5. 仅根据 baseline train 失败结果生成失败归因和候选。 +6. 对每个候选分别完整评测 train 和 validation，校验 case ID 一致，再计算 delta。 +7. 对每个候选执行同一完整 gate，并从已接受候选中按 validation、train、原始顺序稳定选择最佳项。 +8. 写候选、round、评测结果、delta、gate、输入快照哈希和真实耗时到临时 run 目录。 +9. 若没有接受候选或未指定 `--update-source`，记录对应 writeback 状态。 +10. 若允许回写，先确认源文件仍与起始快照一致，再执行原子多文件写入；失败则回滚并记录原因。 +11. 完成 `writeback.json`、最终 JSON/Markdown 后，把临时目录原子重命名为 `runs//`。已存在的 run ID 不允许覆盖。 +12. 根目录 `optimization_report.json` 和 `.md` 仅作为最新结果的便利副本；不可变证据以 run 目录为准。 + +## Fake 模式隔离规则 + +fake model 只能读取用户输入和当前 prompt 文本。evalset 的 expectation、expected answer、split、protected 和标签只能由 evaluator、归因器或 gate 使用。 + +公开样例输入改成包含明确业务值的指令，例如 `intent=refund, priority=high`。fake model 从用户指令解析值：baseline 对严格格式请求加入多余说明，过度候选对所有请求强制 JSON，安全候选只在用户明确要求时使用严格格式。相同输入和 prompt 在 train/validation 中必须产生相同输出。 + +fake optimizer 根据 baseline train 中的失败类别选择规则模板。例如发现 format 与 exact-answer 失败时，生成一个全局严格候选和一个按请求限定的候选。它不得读取 validation 结果或 case ID。 + +## 成本语义 + +`max_total_cost` 是可选 gate。`CostSummary.complete=True` 时，total 必须是已知 optimizer、evaluator 和 agent 成本之和。fake backend 提供完整成本。 + +SDK 无法获得 agent/provider 完整成本时设置 `complete=False`。若配置了成本上限，候选必须以 `cost_unavailable` 拒绝；未配置成本上限时可以继续其他 gate，但报告必须把 SDK 提供的数值标记为 `reported_optimizer_cost`，不能称为完整总成本。 + +## 错误处理 + +- baseline 输入、评测或 prompt 恢复失败：终止运行，不选择候选，不最终回写。 +- 单个候选评测失败：记录候选拒绝原因，继续评测其他候选。 +- case ID 缺失、重复或集合不一致：该候选拒绝。 +- 非有限数值、非标准 JSON、重复 target path、split 冲突：输入阶段直接报错。 +- 最终回写 compare-and-swap 失败：不覆盖源文件，记录并抛出并发修改错误。 +- 多文件写入失败：回滚已写字段；回滚成功时状态为 `rolled_back`，回滚失败时状态为 `rollback_failed` 并报告受影响路径。 +- 审计写入失败：最终回写不会开始。 + +## 审计产物 + +每个 `runs//` 至少包含： + +- `optimization_report.json` 与 `optimization_report.md`； +- `input_hashes.json` 和规范化配置快照； +- `baseline_prompts/`、`candidate_prompts/` 与 `prompt_diffs/`； +- `case_results/` 和 `per_case_deltas.json`； +- `rounds/`，保存每轮候选、理由、成本、耗时和 optimizer 指标； +- `gate_decisions.json`； +- `writeback.json`； +- SDK 模式下的 `sdk_optimizer/` 原始产物。 + +报告路径优先保存仓库相对路径；仓库外输入仅保存用户传入的规范化路径，不写入维护者机器生成样例。示例报告由测试临时生成并与 committed inputs 的哈希自动比对。 + +## 测试设计 + +### Backend 契约测试 + +fake 与 SDK backend 都必须返回完整、case ID 唯一且集合一致的 `EvalResult`。SDK 测试使用真实 `AgentEvaluator`、`AgentOptimizer` 和 `TargetPrompt`；仅 monkeypatch GEPA/外部反思模型调用，沿用仓库现有 facade 测试模式。 + +### 安全回写测试 + +- gate 拒绝且传入 `--update-source` 时，所有源 prompt 字节和哈希保持不变； +- gate 接受时才写入选中候选； +- 多 prompt 第二个字段写入失败时，第一个字段回滚； +- 起始快照后发生并发修改时拒绝覆盖； +- 审计写入失败时不触发回写。 + +### 隐藏决策准确率 + +新增至少 10 个与公开六例分离的 holdout 场景，覆盖安全提升、无提升、训练提升而 validation 退化、protected regression、new hard fail、单 case drop 和超预算。标签与 pipeline 输入分离，计算 `correct / total` 并断言 `>= 0.80`。 + +### 失败归因准确率 + +新增独立归因语料，覆盖 format、final response、tool、parameter、rubric 和 knowledge 类别。归因器只接收评测错误和证据，不接收标签；断言准确率 `>= 0.75`，且每个失败都有非空 reason 与 evidence。 + +### 性能与报告 + +用 wall clock 运行完整 fake+trace pipeline，断言少于 180 秒；报告中的 duration 必须大于零且不大于测试观测值的合理上界。测试重新生成 example report，校验输入哈希、相对路径、严格 JSON、全部必需字段和旧 run 不被覆盖。 + +## 兼容性与迁移 + +- 保留现有 CLI 参数和同步 `run_pipeline()`；新增异步入口不破坏已有调用方。 +- fake 和 SDK 输入统一使用官方 SDK EvalSet 形状；loader 在本次 PR 内继续兼容旧 `cases` 形状，但 README 和 committed examples 只展示官方形状。 +- JSON schema 版本提升，新增字段不复用旧字段表达不同语义。 +- `--update-source` 的用户语义保持不变，但实际执行从 optimizer 内部提前写入改为完整 gate 后写入。 + +## 合并标准 + +以下条件全部满足后才建议合并： + +1. 六个公开 case 完整运行并生成 JSON、Markdown 和审计目录； +2. SDK 路径提供真实 baseline/candidate 逐 case 结果，不存在可被接受的 partial gate； +3. 训练提升而 validation 退化的候选被拒绝； +4. holdout 决策准确率至少 80%； +5. 独立归因准确率至少 75%，所有失败可解释； +6. fake+trace wall clock 少于三分钟且报告记录真实耗时； +7. gate 拒绝、评测失败、审计失败和并发修改时源 prompt 不被覆盖； +8. committed example report 的哈希与 committed inputs 一致且没有贡献者绝对路径； +9. 目标测试、完整示例测试、仓库 lint、build 和 CI 全部通过。 From 9642a85633beeb540b9e806a7b95100442675bb5 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 12:45:38 +0800 Subject: [PATCH 11/34] docs: plan issue 91 safe optimization loop --- .../2026-07-10-issue-91-safe-closed-loop.md | 1884 +++++++++++++++++ 1 file changed, 1884 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-10-issue-91-safe-closed-loop.md diff --git a/docs/superpowers/plans/2026-07-10-issue-91-safe-closed-loop.md b/docs/superpowers/plans/2026-07-10-issue-91-safe-closed-loop.md new file mode 100644 index 00000000..a7abd5fb --- /dev/null +++ b/docs/superpowers/plans/2026-07-10-issue-91-safe-closed-loop.md @@ -0,0 +1,1884 @@ +# Issue #91 Safe Closed-Loop Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build one auditable Evaluation + Optimization pipeline that satisfies Issue #91, rejects incomplete or regressing candidates, and writes source prompts only after the complete gate accepts them. + +**Architecture:** Move orchestration from `run_pipeline.py` into a shared async pipeline used by both fake and SDK backends. Backends return the same complete schemas; the pipeline owns evaluation order, gate decisions, run-specific audit persistence, and post-gate transactional writeback. + +**Tech Stack:** Python 3.11, dataclasses, asyncio, pathlib, `AgentEvaluator`, `AgentOptimizer`, `TargetPrompt`, pytest, YAPF, flake8. + +--- + +## Working-tree rules + +This worktree already contains intentional uncommitted changes in 17 Issue #91 files. Preserve them. Do not reset, restore, stash, or bulk-stage the worktree. Every task below stages only the listed files and commits one coherent change. + +### Task 1: Establish the baseline and lock down data contracts + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/schemas.py:12-157` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/loader.py:14-142` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/config.py:53-155` +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:771-814` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_config_validation.py` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py` + +- [ ] **Step 1: Record the current baseline without changing files** + +Run: + +```powershell +git status --short +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +``` + +Expected: the pre-existing Issue #91 files remain modified and the current suite reports `80 passed`. + +- [ ] **Step 2: Add failing strict-input and path-collision tests** + +Append these tests to `test_config_validation.py`: + +```python +import math + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.config import parse_optimizer_config +from examples.optimization.eval_optimize_loop.eval_loop.loader import read_json +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.run_pipeline import _load_sdk_gate_config + + +@pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity"]) +def test_read_json_rejects_non_standard_constants(tmp_path: Path, constant: str): + path = tmp_path / "bad.json" + path.write_text('{"value": ' + constant + "}", encoding="utf-8") + + with pytest.raises(ValueError, match="non-standard JSON constant"): + read_json(path) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf"), True]) +def test_fake_gate_rejects_non_finite_or_boolean_numbers(value): + with pytest.raises(ValueError, match="finite number"): + parse_optimizer_config( + {"gate": {"min_val_score_improvement": value}}, + path="optimizer.json", + ) + + +def test_sdk_gate_allows_explicitly_disabled_cost_limit(tmp_path: Path): + path = tmp_path / "gate.json" + path.write_text('{"gate": {"max_total_cost": null}}', encoding="utf-8") + + config = _load_sdk_gate_config(path) + + assert config["max_total_cost"] is None + + +def test_eval_case_rejects_explicit_split_mismatch(): + with pytest.raises(ValueError, match="split mismatch"): + EvalCase.from_dict( + { + "id": "case-1", + "split": "train", + "input": "Return OK", + "expectation": {"type": "exact", "expected": "OK"}, + }, + split="validation", + ) +``` + +Append this test to `test_sdk_backend.py`: + +```python +def test_target_prompt_paths_reject_same_resolved_file(tmp_path: Path): + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="same resolved file"): + _parse_target_prompt_paths( + [ + f"system_prompt={prompt_path}", + f"router_prompt={prompt_path.parent / '.' / prompt_path.name}", + ], + default_prompt_path=prompt_path, + ) +``` + +- [ ] **Step 3: Run the new tests and confirm the current implementation fails** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_config_validation.py examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py::test_target_prompt_paths_reject_same_resolved_file -v +``` + +Expected: failures show that Python JSON accepts non-standard constants, fake gate numbers accept `NaN`, split mismatch is not rejected, and aliased target paths are accepted. + +- [ ] **Step 4: Extend the shared schemas without breaking existing callers** + +In `schemas.py`, add defaults after the existing non-default `CaseResult` fields and add the new audit models: + +```python +from typing import Any +from typing import Literal + + +@dataclass(frozen=True) +class CaseResult: + case_id: str + split: str + score: float + passed: bool + output: str + metrics: dict[str, float] = field(default_factory=dict) + trace: dict[str, Any] = field(default_factory=dict) + trace_available: bool = False + failure_category: str | None = None + failure_reason: str | None = None + evidence: str | None = None + cost: float = 0.0 + hard_failed: bool = False + expected_failure_category: str | None = None + + +@dataclass(frozen=True) +class CandidatePrompt: + candidate_id: str + prompt: str + rationale: str + prompt_diff: str + prompt_fields: dict[str, str] = field(default_factory=dict) + + def bundle(self) -> dict[str, str]: + if self.prompt_fields: + return dict(self.prompt_fields) + return {"system_prompt": self.prompt} + + +@dataclass(frozen=True) +class CostSummary: + optimizer: float = 0.0 + evaluator: float = 0.0 + agent: float = 0.0 + total: float = 0.0 + complete: bool = True + + +@dataclass(frozen=True) +class OptimizationRound: + round_id: str + candidate_id: str + prompts: dict[str, str] + rationale: str + metrics: dict[str, float] + cost: float + duration_seconds: float + + +WritebackStatus = Literal[ + "rejected", + "not_requested", + "applied", + "rolled_back", + "rollback_failed", +] + + +@dataclass(frozen=True) +class WritebackResult: + status: WritebackStatus + before_hashes: dict[str, str] = field(default_factory=dict) + after_hashes: dict[str, str] = field(default_factory=dict) + error: str | None = None + + +@dataclass(frozen=True) +class OptimizationResult: + candidates: list[CandidatePrompt] + rounds: list[OptimizationRound] + cost: CostSummary + raw_summary: dict[str, Any] = field(default_factory=dict) +``` + +Add `rounds`, `cost_summary`, and `writeback` to the end of `OptimizationReport`, with defaults so existing report construction remains valid during the refactor: + +```python + rounds: list[OptimizationRound] = field(default_factory=list) + cost_summary: CostSummary = field(default_factory=CostSummary) + writeback: WritebackResult = field( + default_factory=lambda: WritebackResult(status="not_requested") + ) +``` + +- [ ] **Step 5: Implement strict JSON, finite-number, split, and path validation** + +Replace `read_json()` in `loader.py` with: + +```python +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-standard JSON constant is not allowed: {value}") + + +def read_json(path: str | Path) -> dict[str, Any]: + resolved = Path(path) + try: + with resolved.open("r", encoding="utf-8") as file: + payload = json.load(file, parse_constant=_reject_json_constant) + except (json.JSONDecodeError, ValueError) as exc: + raise ValueError(f"invalid strict JSON in {resolved}: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object in {resolved}") + return payload +``` + +In `config.py`, use one helper for every numeric gate field: + +```python +def _finite_number(value: Any, *, field_name: str, minimum: float, maximum: float | None = None) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field_name} must be a finite number") + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"{field_name} must be a finite number") + if parsed < minimum or (maximum is not None and parsed > maximum): + upper = f" and <= {maximum}" if maximum is not None else "" + raise ValueError(f"{field_name} must be >= {minimum}{upper}") + return parsed +``` + +Use it for `min_val_score_improvement` and `max_score_drop_per_case`. Keep fake `GateConfig.max_total_cost` defaulted to `1.0`, but type it as `float | None`; accept `None` as “cost gate disabled” and otherwise validate it with `_finite_number()`. Apply the same rule in `_load_sdk_gate_config()`. In `EvalCase.from_dict()`, reject an explicit payload split that differs from the loader split. In `_parse_target_prompt_paths()`, track `Path(path).resolve()` values and raise `ValueError("--target-prompt fields must not reference the same resolved file")` on duplicates. + +- [ ] **Step 6: Run focused and full example tests** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_config_validation.py examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py::test_target_prompt_paths_reject_same_resolved_file -v +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +``` + +Expected: all tests pass and no existing example test regresses. + +- [ ] **Step 7: Commit the contract and validation change** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/schemas.py examples/optimization/eval_optimize_loop/eval_loop/loader.py examples/optimization/eval_optimize_loop/eval_loop/config.py examples/optimization/eval_optimize_loop/run_pipeline.py examples/optimization/eval_optimize_loop/tests/test_config_validation.py examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +git commit -m "refactor(examples): define safe optimization contracts" +``` + +### Task 2: Add transactional prompt snapshots and post-gate writeback + +**Files:** +- Create: `examples/optimization/eval_optimize_loop/eval_loop/writeback.py` +- Create: `examples/optimization/eval_optimize_loop/tests/test_writeback.py` + +- [ ] **Step 1: Write failing snapshot, rollback, and compare-and-swap tests** + +Create `test_writeback.py`: + +```python +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ConcurrentPromptUpdateError +from examples.optimization.eval_optimize_loop.eval_loop.writeback import commit_prompt_bundle +from examples.optimization.eval_optimize_loop.eval_loop.writeback import snapshot_prompt_files +from examples.optimization.eval_optimize_loop.eval_loop.writeback import temporary_prompt_bundle + + +def _files(tmp_path: Path) -> dict[str, Path]: + system = tmp_path / "system.txt" + router = tmp_path / "router.txt" + system.write_text("system baseline", encoding="utf-8") + router.write_text("router baseline", encoding="utf-8") + return {"system_prompt": system, "router_prompt": router} + + +def test_temporary_prompt_bundle_always_restores_original_bytes(tmp_path: Path): + paths = _files(tmp_path) + snapshot = snapshot_prompt_files(paths) + + with pytest.raises(RuntimeError, match="candidate failed"): + with temporary_prompt_bundle( + snapshot, + {"system_prompt": "candidate system", "router_prompt": "candidate router"}, + ): + assert paths["system_prompt"].read_text(encoding="utf-8") == "candidate system" + raise RuntimeError("candidate failed") + + assert paths["system_prompt"].read_bytes() == b"system baseline" + assert paths["router_prompt"].read_bytes() == b"router baseline" + + +def test_commit_rejects_concurrent_source_change(tmp_path: Path): + paths = _files(tmp_path) + snapshot = snapshot_prompt_files(paths) + paths["system_prompt"].write_text("changed by another process", encoding="utf-8") + + with pytest.raises(ConcurrentPromptUpdateError): + commit_prompt_bundle( + snapshot, + {"system_prompt": "candidate system", "router_prompt": "candidate router"}, + ) + + assert paths["system_prompt"].read_text(encoding="utf-8") == "changed by another process" + + +def test_second_replace_failure_rolls_back_first_file(tmp_path: Path, monkeypatch): + paths = _files(tmp_path) + snapshot = snapshot_prompt_files(paths) + real_replace = os.replace + calls = 0 + + def fail_second_replace(source, destination): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("second replace failed") + return real_replace(source, destination) + + monkeypatch.setattr(os, "replace", fail_second_replace) + result = commit_prompt_bundle( + snapshot, + {"system_prompt": "candidate system", "router_prompt": "candidate router"}, + ) + + assert result.status == "rolled_back" + assert paths["system_prompt"].read_bytes() == b"system baseline" + assert paths["router_prompt"].read_bytes() == b"router baseline" +``` + +- [ ] **Step 2: Run the tests and confirm the module is missing** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_writeback.py -v +``` + +Expected: collection fails because `eval_loop.writeback` does not exist. + +- [ ] **Step 3: Implement the writeback module** + +Create `writeback.py` with these public types and functions: + +```python +from __future__ import annotations + +import hashlib +import os +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +from .schemas import WritebackResult + + +class ConcurrentPromptUpdateError(RuntimeError): + pass + + +@dataclass(frozen=True) +class PromptFileSnapshot: + name: str + path: Path + content: bytes + sha256: str + + +@dataclass(frozen=True) +class PromptSnapshot: + files: dict[str, PromptFileSnapshot] + + def hashes(self) -> dict[str, str]: + return {name: item.sha256 for name, item in self.files.items()} + + +def _hash_bytes(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def snapshot_prompt_files(paths: dict[str, str | Path]) -> PromptSnapshot: + files = {} + for name, raw_path in paths.items(): + path = Path(raw_path) + content = path.read_bytes() + files[name] = PromptFileSnapshot( + name=name, + path=path, + content=content, + sha256=_hash_bytes(content), + ) + return PromptSnapshot(files=files) + + +def _current_hashes(snapshot: PromptSnapshot) -> dict[str, str]: + return { + name: _hash_bytes(item.path.read_bytes()) + for name, item in snapshot.files.items() + } + + +def _atomic_replace_bytes(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temp_path = Path(temp_name) + try: + with os.fdopen(descriptor, "wb") as file: + file.write(content) + file.flush() + os.fsync(file.fileno()) + os.replace(temp_path, path) + finally: + if temp_path.exists(): + temp_path.unlink() + + +def _restore_snapshot(snapshot: PromptSnapshot) -> list[str]: + failures = [] + for name, item in snapshot.files.items(): + try: + _atomic_replace_bytes(item.path, item.content) + except OSError as exc: + failures.append(f"{name}: {exc}") + return failures + + +@contextmanager +def temporary_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> Iterator[None]: + missing = sorted(set(snapshot.files) - set(prompts)) + if missing: + raise ValueError(f"candidate prompt bundle is missing fields: {missing}") + try: + for name, item in snapshot.files.items(): + _atomic_replace_bytes(item.path, prompts[name].encode("utf-8")) + yield + finally: + failures = _restore_snapshot(snapshot) + if failures: + raise RuntimeError(f"failed to restore prompt snapshot: {failures}") + if _current_hashes(snapshot) != snapshot.hashes(): + raise RuntimeError("restored prompt hashes do not match the original snapshot") + + +def commit_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> WritebackResult: + before_hashes = _current_hashes(snapshot) + if before_hashes != snapshot.hashes(): + raise ConcurrentPromptUpdateError("source prompts changed after the run snapshot") + try: + for name, item in snapshot.files.items(): + _atomic_replace_bytes(item.path, prompts[name].encode("utf-8")) + except (OSError, KeyError) as exc: + rollback_failures = _restore_snapshot(snapshot) + status = "rollback_failed" if rollback_failures else "rolled_back" + return WritebackResult( + status=status, + before_hashes=before_hashes, + after_hashes=_current_hashes(snapshot), + error=f"{exc}; rollback_failures={rollback_failures}", + ) + return WritebackResult( + status="applied", + before_hashes=before_hashes, + after_hashes=_current_hashes(snapshot), + ) +``` + +- [ ] **Step 4: Run writeback tests** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_writeback.py -v +``` + +Expected: all three tests pass. + +- [ ] **Step 5: Commit the transactional writeback component** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/writeback.py examples/optimization/eval_optimize_loop/tests/test_writeback.py +git commit -m "feat(examples): add transactional prompt writeback" +``` + +### Task 3: Normalize fake and SDK backend contracts + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/backends.py` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/evaluator.py` + +- [ ] **Step 1: Replace the unsafe writeback expectation and add result-mapping tests** + +Replace `test_sdk_backend_passes_update_source_true` with: + +```python +def test_sdk_backend_never_delegates_source_writeback(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["update_source"] is False + assert prompt_path.read_text(encoding="utf-8") == "baseline" +``` + +Add a mapping test using small fake SDK result objects: + +```python +def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): + metric = SimpleNamespace( + metric_name="final_response_avg_score", + score=0.5, + eval_status="FAILED", + details=SimpleNamespace(reason="response mismatch"), + ) + invocation = SimpleNamespace( + actual_invocation=SimpleNamespace( + user_content={"parts": [{"text": "query"}]}, + final_response={"parts": [{"text": "actual"}]}, + intermediate_data={"tool_calls": [{"name": "lookup"}]}, + ) + ) + run = SimpleNamespace( + final_eval_status="FAILED", + overall_eval_metric_results=[metric], + eval_metric_result_per_invocation=[invocation], + error_message=None, + ) + result = SimpleNamespace( + results_by_eval_set_id={ + "set": SimpleNamespace(eval_results_by_eval_id={"case-1": [run]}) + } + ) + expected = EvalCase( + case_id="case-1", + split="validation", + input="query", + expectation={"type": "exact", "expected": "expected"}, + expected_failure_category="final_response_mismatch", + ) + + converted = _eval_result_from_sdk_result( + result, + prompt_id="candidate", + split="validation", + expected_cases={"case-1": expected}, + ) + + case = converted.cases[0] + assert case.metrics == {"final_response_avg_score": 0.5} + assert case.trace_available is True + assert case.trace["final_response"] == "actual" + assert case.expected_failure_category == "final_response_mismatch" +``` + +- [ ] **Step 2: Run the focused SDK tests and confirm failures** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py -k "never_delegates or preserves_metrics" -v +``` + +Expected: the old backend passes `update_source=True`, and the SDK converter lacks the new arguments and fields. + +- [ ] **Step 3: Make backend operations async and return `OptimizationResult`** + +In `backends.py`, define protocols and make both backends implement them: + +```python +from typing import Protocol + +from .schemas import OptimizationResult + + +class EvaluationBackend(Protocol): + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + raise NotImplementedError + + +class OptimizationBackend(Protocol): + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + raise NotImplementedError +``` + +Keep the current synchronous `SDKBackend.optimize()` only as a compatibility wrapper around `optimize_async()`. Remove the `update_source` field from `SDKBackend`; in `optimize_async()` hard-code: + +```python + result = await AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target_prompt, + train_dataset_path=str(train_path), + validation_dataset_path=str(val_path), + output_dir=str(output_dir), + update_source=False, + verbose=0, + ) +``` + +Extract all unique round candidates and the final best prompt into `CandidatePrompt.prompt_fields`; use IDs `sdk_round_001`, `sdk_round_002`, and `sdk_best`. Preserve `acceptance_reason`, metric breakdown, round cost, and duration in `OptimizationRound`. + +- [ ] **Step 4: Preserve per-metric results and invocation trace** + +Change `_eval_result_from_sdk_result()` to accept `expected_cases: dict[str, EvalCase]`. For each eval ID: + +```python +def _metric_scores(runs: list[Any]) -> dict[str, float]: + values: dict[str, list[float]] = {} + for run in runs: + for metric in getattr(run, "overall_eval_metric_results", []) or []: + score = getattr(metric, "score", None) + if isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(float(score)): + values.setdefault(str(metric.metric_name), []).append(float(score)) + return {name: round(sum(items) / len(items), 6) for name, items in values.items()} + + +def _invocation_trace(runs: list[Any]) -> tuple[dict[str, Any], bool]: + if not runs: + return {}, False + invocation_results = getattr(runs[-1], "eval_metric_result_per_invocation", []) or [] + if not invocation_results: + return {}, False + actual = getattr(invocation_results[-1], "actual_invocation", None) + if actual is None: + return {}, False + return { + "user_content": _safe_jsonable(getattr(actual, "user_content", None)), + "final_response": _content_text(getattr(actual, "final_response", None)), + "intermediate_data": _safe_jsonable(getattr(actual, "intermediate_data", None)), + }, True +``` + +Set `CaseResult.score` to the mean of the metric map, preserve `expected_failure_category`, and validate that the SDK result case IDs exactly match `expected_cases`. + +- [ ] **Step 5: Run backend tests and the example suite** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py -v +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +``` + +Expected: all tests pass; tests that previously asserted delegated source writes now assert `False`. + +- [ ] **Step 6: Commit the backend normalization** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/backends.py examples/optimization/eval_optimize_loop/eval_loop/evaluator.py examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +git commit -m "refactor(examples): normalize optimization backends" +``` + +### Task 4: Remove fake-model oracle and split leakage + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/fake_model.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/optimizer.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/backends.py` +- Modify: `examples/optimization/eval_optimize_loop/data/train.evalset.json` +- Modify: `examples/optimization/eval_optimize_loop/data/val.evalset.json` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py` + +- [ ] **Step 1: Replace oracle-dependent tests with metamorphic tests** + +Replace `test_fake_model_generalization.py` with tests that hold user input and prompt constant while changing evaluator-only fields: + +```python +from __future__ import annotations + +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.optimizer import FakeOptimizer +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult + + +def _case(*, split: str, expected: str, protected: bool) -> EvalCase: + return EvalCase( + case_id=f"case-{split}-{expected}", + split=split, + input="Return only PUBLIC; do not use JSON.", + expectation={"type": "exact", "expected": expected}, + protected=protected, + tags=["evaluator-only"], + ) + + +def test_fake_model_output_does_not_change_with_oracle_fields(): + model = FakeModel(seed=91) + first, _, _ = model.generate("candidate", "baseline prompt", _case(split="train", expected="SECRET", protected=False)) + second, _, _ = model.generate("candidate", "baseline prompt", _case(split="validation", expected="OTHER", protected=True)) + + assert first == "PUBLIC" + assert second == "PUBLIC" + + +def test_fake_optimizer_requires_observed_training_failures(): + passing = EvalResult( + prompt_id="baseline", + split="train", + score=1.0, + passed=True, + cost=0.0, + cases=[CaseResult(case_id="c1", split="train", score=1.0, passed=True, output="OK")], + ) + failing = EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[ + CaseResult( + case_id="c1", + split="train", + score=0.0, + passed=False, + output="bad", + failure_category="format_violation", + failure_reason="not strict JSON", + ) + ], + ) + + optimizer = FakeOptimizer() + assert optimizer.propose("baseline", passing, {"by_category": {}}) == [] + candidates = optimizer.propose( + "baseline", + failing, + {"by_category": {"format_violation": 1}}, + ) + assert [candidate.candidate_id for candidate in candidates] == [ + "candidate_001_overfit", + "candidate_002_safe", + ] +``` + +- [ ] **Step 2: Run the metamorphic tests and confirm leakage** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py -v +``` + +Expected: outputs differ when split, protected, or expected value changes, and `FakeOptimizer.propose()` has the old signature. + +- [ ] **Step 3: Make fake behavior depend only on input and prompt** + +Replace expectation-driven output helpers in `fake_model.py` with input parsers: + +```python +import json +import re + + +ASSIGNMENT_RE = re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*)=([A-Za-z0-9_-]+)\b") +ONLY_RE = re.compile(r"return only\s+([A-Za-z0-9_-]+)", re.IGNORECASE) + + +def _assignments(text: str) -> dict[str, str]: + return {key: value for key, value in ASSIGNMENT_RE.findall(text)} + + +def _only_value(text: str) -> str | None: + match = ONLY_RE.search(text) + return match.group(1) if match else None + + +class FakeModel: + COST_PER_CALL = 0.001 + + def __init__(self, seed: int = 91) -> None: + self.seed = seed + + def generate(self, prompt_id: str, prompt: str, case: EvalCase) -> tuple[str, dict[str, Any], float]: + mode = self._mode(prompt) + output = self._render(mode, case.input) + return output, { + "seed": self.seed, + "prompt_id": prompt_id, + "prompt_mode": mode, + "case_id": case.case_id, + }, self.COST_PER_CALL + + def _mode(self, prompt: str) -> str: + if "Always force every final answer into JSON" in prompt: + return "overfit" + if "Use strict JSON only when the user explicitly asks" in prompt: + return "safe" + return "baseline" + + def _render(self, mode: str, user_input: str) -> str: + assignments = _assignments(user_input) + only_value = _only_value(user_input) + asks_json = "strict json" in user_input.lower() + natural = self._natural_answer(user_input) + if mode == "overfit": + payload = assignments or {"answer": only_value or natural} + return json.dumps(payload, sort_keys=True) + if asks_json: + payload = json.dumps(assignments, sort_keys=True) + return payload if mode == "safe" else f"Here is the JSON you requested: {payload}" + if only_value is not None: + return only_value + return natural + + def _natural_answer(self, user_input: str) -> str: + lowered = user_input.lower() + if "latency" in lowered and "retries" in lowered: + return "Latency and retries need monitoring." + if "cache" in lowered and "stale data" in lowered: + return "Cache invalidation prevents stale data." + return "The request was handled naturally." +``` + +Delete all reads of `case.expectation`, `case.split`, `case.protected`, `case.tags`, and `case.simulated_outputs` from `FakeModel`. + +- [ ] **Step 4: Make fake optimization consume baseline failures** + +Change `FakeOptimizer.propose()` to: + +```python +def propose( + self, + baseline_prompt: str, + baseline_train: EvalResult, + failure_summary: dict[str, object], +) -> list[CandidatePrompt]: + failed_categories = { + case.failure_category + for case in baseline_train.cases + if not case.passed and case.failure_category + } + summarized = set((failure_summary.get("by_category") or {}).keys()) + if not (failed_categories | summarized): + return [] + if not ((failed_categories | summarized) & {"format_violation", "final_response_mismatch"}): + return [] + return self._format_candidates(baseline_prompt) +``` + +Move the two existing candidate constructors into `_format_candidates()`, remove `OPTIMIZER_MARKER`, and keep the natural-language instructions used by `FakeModel._mode()`. + +- [ ] **Step 5: Update the six public user inputs without exposing judge labels to the model** + +Keep official SDK EvalSet shape. Use these user texts and evaluator expectations: + +```json +{ + "train": [ + ["Return strict JSON with intent=refund and priority=high.", {"type": "json", "expected_values": {"intent": "refund", "priority": "high"}}], + ["Return strict JSON with status=READY and next_step=ship.", {"type": "json", "expected_values": {"status": "READY", "next_step": "ship"}}], + ["Explain latency and retries naturally in under 80 characters.", {"type": "rubric", "must_include": ["latency", "retries"], "max_chars": 80}] + ], + "validation": [ + ["Return strict JSON with status=approved and next_step=email_customer.", {"type": "json", "expected_values": {"status": "approved", "next_step": "email_customer"}}], + ["Explain cache invalidation and stale data naturally.", {"type": "rubric", "must_include": ["cache", "stale data"], "forbidden": ["{", "}", "json"]}], + ["Return only YES; do not use JSON.", {"type": "exact", "expected": "YES"}] + ] +} +``` + +Mark the final validation case protected. This yields baseline train `1/3`, overfit train `2/3`, baseline validation `2/3`, overfit validation `1/3`, and safe validation `3/3` without reading split. + +- [ ] **Step 6: Run fake model and pipeline tests** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py -v +``` + +Expected: same input/prompt produces the same output across oracle-field changes; the overfit candidate is rejected and the safe candidate is selected. + +- [ ] **Step 7: Commit the oracle-free fake backend** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/fake_model.py examples/optimization/eval_optimize_loop/eval_loop/optimizer.py examples/optimization/eval_optimize_loop/eval_loop/backends.py examples/optimization/eval_optimize_loop/data/train.evalset.json examples/optimization/eval_optimize_loop/data/val.evalset.json examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py +git commit -m "fix(examples): remove fake evaluation oracle leakage" +``` + +### Task 5: Build the shared async pipeline and complete gate + +**Files:** +- Create: `examples/optimization/eval_optimize_loop/eval_loop/pipeline.py` +- Create: `examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/gate.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/report.py` +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py` + +- [ ] **Step 1: Write orchestration tests with recording backends** + +Create `test_pipeline_orchestration.py` with the following complete recording backend. It uses temporary official-shaped input files and records every backend call without patching production code: + +```python +def _case(case_id: str, split: str, score: float) -> CaseResult: + return CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="OK" if score >= 1.0 else "FAIL", + metrics={"exact_match": score}, + trace={"final_response": "OK" if score >= 1.0 else "FAIL"}, + trace_available=True, + hard_failed=score == 0.0, + cost=0.01, + ) + + +def _result(prompt_id: str, split: str, scores: list[float]) -> EvalResult: + cases = [_case(f"{split[0]}{index}", split, score) for index, score in enumerate(scores)] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=sum(scores) / len(scores), + passed=all(case.passed for case in cases), + cost=sum(case.cost for case in cases), + cases=cases, + ) + + +class RecordingBackend: + def __init__( + self, + tmp_path: Path, + *, + candidate_regresses: bool = False, + cost_complete: bool = True, + ) -> None: + self.calls: list[tuple[str, ...]] = [] + self.candidate_regresses = candidate_regresses + self.cost_complete = cost_complete + self.train_path = DEFAULT_TRAIN + self.val_path = DEFAULT_VAL + self.config_path = DEFAULT_OPTIMIZER_CONFIG + self.prompt_path = tmp_path / "prompt.txt" + self.prompt_path.write_text("baseline prompt", encoding="utf-8") + + async def evaluate(self, *, prompt_id: str, split: str, **kwargs: Any) -> EvalResult: + self.calls.append(("evaluate", prompt_id, split)) + if prompt_id == "baseline": + scores = [0.0, 1.0, 1.0] + elif self.candidate_regresses and split == "validation": + scores = [0.0, 0.0, 1.0] + else: + scores = [1.0, 1.0, 1.0] + return _result(prompt_id, split, scores) + + async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: + self.calls.append(("optimize",)) + prompt = "candidate safe prompt" + candidate = CandidatePrompt( + candidate_id="candidate_safe", + prompt=prompt, + rationale="fix observed baseline failure", + prompt_diff="-baseline prompt\n+candidate safe prompt", + prompt_fields={"system_prompt": prompt}, + ) + return OptimizationResult( + candidates=[candidate], + rounds=[ + OptimizationRound( + round_id="round_001", + candidate_id=candidate.candidate_id, + prompts=candidate.bundle(), + rationale=candidate.rationale, + metrics={"discovery_score": 1.0}, + cost=0.02, + duration_seconds=0.01, + ) + ], + cost=CostSummary(optimizer=0.02, total=0.02, complete=self.cost_complete), + ) + + +async def _run_recording_pipeline(tmp_path: Path, backend: RecordingBackend) -> OptimizationReport: + return await run_pipeline_async( + train_path=backend.train_path, + val_path=backend.val_path, + optimizer_config_path=backend.config_path, + prompt_path=backend.prompt_path, + output_dir=tmp_path / "out", + mode="fake", + backend=backend, + run_id="recording-run", + ) + + +@pytest.mark.asyncio +async def test_pipeline_evaluates_baselines_before_optimization_and_every_candidate(tmp_path: Path): + backend = RecordingBackend(tmp_path) + report = await run_pipeline_async( + train_path=backend.train_path, + val_path=backend.val_path, + optimizer_config_path=backend.config_path, + prompt_path=backend.prompt_path, + output_dir=tmp_path / "out", + mode="fake", + backend=backend, + ) + + assert backend.calls == [ + ("evaluate", "baseline", "train"), + ("evaluate", "baseline", "validation"), + ("optimize",), + ("evaluate", "candidate_safe", "train"), + ("evaluate", "candidate_safe", "validation"), + ] + assert report.selected_candidate == "candidate_safe" + + +@pytest.mark.asyncio +async def test_gate_rejection_never_writes_source_even_when_requested(tmp_path: Path): + backend = RecordingBackend(tmp_path, candidate_regresses=True) + original = backend.prompt_path.read_bytes() + + report = await run_pipeline_async( + train_path=backend.train_path, + val_path=backend.val_path, + optimizer_config_path=backend.config_path, + prompt_path=backend.prompt_path, + output_dir=tmp_path / "out", + mode="fake", + update_source=True, + backend=backend, + ) + + assert report.selected_candidate is None + assert report.writeback.status == "rejected" + assert backend.prompt_path.read_bytes() == original + + +@pytest.mark.asyncio +async def test_incomplete_cost_rejects_when_budget_is_configured(tmp_path: Path): + backend = RecordingBackend(tmp_path, cost_complete=False) + report = await _run_recording_pipeline(tmp_path, backend) + + decision = report.gate_decisions[0] + assert decision.accepted is False + assert "cost_unavailable" in decision.reasons +``` + +Import `Any`, all schema classes used above, `DEFAULT_TRAIN`, `DEFAULT_VAL`, `DEFAULT_OPTIMIZER_CONFIG`, and `run_pipeline_async` at the top of the test. The backend method is named `optimize_candidates()` deliberately so it cannot be confused with the retained synchronous `SDKBackend.optimize()` compatibility wrapper. + +- [ ] **Step 2: Run orchestration tests and confirm the shared pipeline is absent** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py -v +``` + +Expected: collection fails because `run_pipeline_async` and `eval_loop.pipeline` do not exist. + +- [ ] **Step 3: Add the request model and shared candidate loop** + +Create `pipeline.py` with: + +```python +@dataclass(frozen=True) +class PipelineRequest: + train_path: Path + validation_path: Path + optimizer_config_path: Path + output_dir: Path + target_prompt_paths: dict[str, Path] + gate_config: dict[str, Any] + trace: bool + update_source: bool + mode: str + run_id: str + + +async def execute_pipeline( + request: PipelineRequest, + *, + evaluator: EvaluationBackend, + optimizer: OptimizationBackend, +) -> OptimizationReport: + started = time.perf_counter() + prompt_snapshot = snapshot_prompt_files(request.target_prompt_paths) + baseline_prompts = { + name: item.content.decode("utf-8") + for name, item in prompt_snapshot.files.items() + } + baseline_train = await evaluator.evaluate( + prompt_id="baseline", + prompts=baseline_prompts, + dataset_path=request.train_path, + split="train", + trace=request.trace, + artifact_dir=request.output_dir / "evaluator" / "baseline_train", + ) + baseline_validation = await evaluator.evaluate( + prompt_id="baseline", + prompts=baseline_prompts, + dataset_path=request.validation_path, + split="validation", + trace=request.trace, + artifact_dir=request.output_dir / "evaluator" / "baseline_validation", + ) + failure_summary = summarize_failures([baseline_train, baseline_validation]) + optimization = await optimizer.optimize_candidates( + baseline_prompts=baseline_prompts, + baseline_train=baseline_train, + failure_summary=failure_summary, + train_path=request.train_path, + validation_path=request.validation_path, + config_path=request.optimizer_config_path, + artifact_dir=request.output_dir / "optimizer", + ) + gate = AcceptanceGate(request.gate_config) + candidate_records = [] + deltas = [] + decisions = [] + cumulative_cost = round( + baseline_train.cost + baseline_validation.cost + optimization.cost.total, + 6, + ) + for candidate in optimization.candidates: + train_result = await evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=candidate.bundle(), + dataset_path=request.train_path, + split="train", + trace=request.trace, + artifact_dir=request.output_dir / "evaluator" / f"{candidate.candidate_id}_train", + ) + validation_result = await evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=candidate.bundle(), + dataset_path=request.validation_path, + split="validation", + trace=request.trace, + artifact_dir=request.output_dir / "evaluator" / f"{candidate.candidate_id}_validation", + ) + candidate_deltas = compute_case_deltas( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + ) + decision = gate.decide( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + deltas=candidate_deltas, + cumulative_cost=cumulative_cost, + cost_summary=optimization.cost, + ) + candidate_records.append({ + "candidate": candidate, + "train_result": train_result, + "validation_result": validation_result, + }) + deltas.extend(candidate_deltas) + decisions.append(decision) + cumulative_cost = decision.total_run_cost + selected_candidate = select_candidate(candidate_records, decisions) + duration_seconds = time.perf_counter() - started + explicit_evaluator_cost = sum( + result.cost + for result in [baseline_train, baseline_validation] + + [record[key] for record in candidate_records for key in ("train_result", "validation_result")] + ) + run_cost = replace( + optimization.cost, + evaluator=round(optimization.cost.evaluator + explicit_evaluator_cost, 6), + total=round(cumulative_cost, 6), + ) + report = build_report( + run={ + "run_id": request.run_id, + "mode": request.mode, + "trace": request.trace, + "update_source": request.update_source, + }, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + per_case_deltas=deltas, + gate_decisions=decisions, + selected_candidate=selected_candidate, + audit={ + "duration_seconds": duration_seconds, + "input_hashes": hash_pipeline_inputs(request, prompt_snapshot), + "cost": to_jsonable(run_cost), + }, + ) + return replace(report, rounds=optimization.rounds, cost_summary=run_cost) +``` + +Implement `hash_pipeline_inputs()` by hashing the train, validation, optimizer config, and snapshotted prompt bytes with SHA-256. Implement `select_candidate()` with the existing stable ordering: accepted candidates only, then highest validation score, then train score, then earliest candidate. + +- [ ] **Step 4: Make the gate reject incomplete data and incomplete configured cost** + +Add `cost_summary: CostSummary` to `AcceptanceGate.decide()`. Before score checks: + +```python +for baseline, candidate in ( + (baseline_train, candidate_train), + (baseline_validation, candidate_validation), +): + baseline_ids = [case.case_id for case in baseline.cases] + candidate_ids = [case.case_id for case in candidate.cases] + if len(baseline_ids) != len(set(baseline_ids)) or len(candidate_ids) != len(set(candidate_ids)): + reasons.append("reject: duplicate case IDs prevent a complete gate") + elif set(baseline_ids) != set(candidate_ids): + reasons.append("reject: baseline and candidate case IDs do not match") + +if self.config.get("max_total_cost") is not None and not cost_summary.complete: + reasons.append("reject: cost_unavailable for configured max_total_cost") +``` + +Treat `cumulative_cost` as the already-incurred baseline plus optimizer plus prior-candidate cost, and add only the current candidate's two explicit evaluation costs when calculating `total_run_cost`. `cost_summary` is passed separately to carry completeness; do not add its total a second time. Use the already-correct soft-to-hard condition: a candidate hard failure is new whenever the baseline case was not hard-failed. + +- [ ] **Step 5: Make `run_pipeline.py` a compatibility wrapper** + +Expose the full async signature and keep the sync API: + +```python +async def run_pipeline_async( + *, + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, + run_id: str | None = None, + backend: Any | None = None, +) -> OptimizationReport: + request, selected_backend = build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=output_dir, + mode=mode, + fake_model=fake_model, + fake_judge=fake_judge, + trace=trace, + sdk_call_agent=sdk_call_agent, + update_source=update_source, + gate_config_path=gate_config_path, + target_prompts=target_prompts, + run_id=run_id, + backend=backend, + ) + return await execute_pipeline( + request, + evaluator=selected_backend, + optimizer=selected_backend, + ) + + +def run_pipeline(**kwargs: Any) -> OptimizationReport: + if _has_running_loop(): + raise ValueError("run_pipeline() cannot run inside an active event loop; await run_pipeline_async()") + return asyncio.run(run_pipeline_async(**kwargs)) +``` + +Delete `_build_sdk_report()` and `_sdk_gate_decision()` after all callers move to `execute_pipeline()`. + +Define `build_pipeline_request_and_backend()` immediately below the wrapper. Its implementation must perform these exact operations in order: + +```python +def build_pipeline_request_and_backend(*, backend: Any | None, **options: Any) -> tuple[PipelineRequest, Any]: + mode = str(options["mode"]) + if mode not in {"fake", "sdk"}: + raise ValueError("mode must be 'fake' or 'sdk'") + prompt_path = Path(options["prompt_path"]) + target_paths = _parse_target_prompt_paths( + options.get("target_prompts"), + default_prompt_path=prompt_path, + ) + selected_run_id = validate_run_id(options.get("run_id") or create_run_id()) + if mode == "fake": + config = load_optimizer_config(options["optimizer_config_path"]) + gate_config = asdict(config.gate) + selected_backend = backend or FakeBackend( + fake_model=options["fake_model"], + fake_judge=options["fake_judge"], + ) + else: + if not options.get("sdk_call_agent") and backend is None: + raise ValueError("--sdk-call-agent is required in sdk mode") + gate_config = _load_sdk_gate_config(options.get("gate_config_path")) + selected_backend = backend or SDKBackend( + target_prompt_paths=target_paths, + call_agent_path=options["sdk_call_agent"], + ) + request = PipelineRequest( + train_path=Path(options["train_path"]), + validation_path=Path(options["val_path"]), + optimizer_config_path=Path(options["optimizer_config_path"]), + output_dir=Path(options["output_dir"]), + target_prompt_paths=target_paths, + gate_config=gate_config, + trace=bool(options["trace"]), + update_source=bool(options["update_source"]), + mode=mode, + run_id=selected_run_id, + ) + return request, selected_backend +``` + +Keep `create_run_id()` and `validate_run_id()` small and deterministic at the boundary: generated IDs use UTC timestamp plus a random suffix; supplied IDs must satisfy the report artifact-name regex. Adapt constructor keywords to the final backend classes, but do not add mode-specific orchestration outside this factory. + +- [ ] **Step 6: Persist audit before optional source write, then attach `WritebackResult`** + +At the end of `execute_pipeline()`, call `prepare_run_artifacts(report, request.output_dir)` before any final write. It creates the run-specific temporary directory, writes the complete pre-write report and audit payload with `allow_nan=False`, and returns `RunArtifactPaths`. Then: + +```python +artifact_paths = prepare_run_artifacts(report, request.output_dir) +if selected_candidate is None: + writeback = WritebackResult(status="rejected", before_hashes=prompt_snapshot.hashes()) +elif not request.update_source: + writeback = WritebackResult(status="not_requested", before_hashes=prompt_snapshot.hashes()) +else: + selected = next(item for item in optimization.candidates if item.candidate_id == selected_candidate) + writeback = commit_prompt_bundle(prompt_snapshot, selected.bundle()) +report = replace(report, writeback=writeback) +finalize_run_artifacts(report, artifact_paths) +return report +``` + +Use `artifact_paths = prepare_run_artifacts(...)` for the value passed to finalization. Task 6 supplies the exact `RunArtifactPaths`, preparation, and finalization implementations. This order is the regression guarantee: audit preparation must succeed before `commit_prompt_bundle()` is reachable. + +- [ ] **Step 7: Run orchestration, gate, fake, and SDK tests** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py examples/optimization/eval_optimize_loop/tests/test_gate.py -v +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +``` + +Expected: complete call order is enforced, rejected candidates never change source bytes, and no test expects `partial_applied`. + +- [ ] **Step 8: Commit the shared pipeline** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/pipeline.py examples/optimization/eval_optimize_loop/eval_loop/gate.py examples/optimization/eval_optimize_loop/eval_loop/report.py examples/optimization/eval_optimize_loop/run_pipeline.py examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py examples/optimization/eval_optimize_loop/tests/test_gate.py +git commit -m "refactor(examples): unify evaluation optimization pipeline" +``` + +### Task 6: Make report artifacts immutable and reproducible + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/report.py` +- Create: `examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py` + +- [ ] **Step 1: Write failing run-isolation and sample-hash tests** + +Create `test_report_artifacts.py`: + +```python +def test_existing_run_id_is_never_overwritten(tmp_path: Path): + first = run_pipeline(output_dir=tmp_path, mode="fake", trace=True, run_id="fixed-run") + run_report = tmp_path / "runs" / "fixed-run" / "optimization_report.json" + original = run_report.read_bytes() + + with pytest.raises(FileExistsError, match="fixed-run"): + run_pipeline(output_dir=tmp_path, mode="fake", trace=True, run_id="fixed-run") + + assert run_report.read_bytes() == original + assert first.audit["duration_seconds"] > 0 + + +def test_committed_example_hashes_match_committed_inputs(): + root = Path("examples/optimization/eval_optimize_loop") + payload = json.loads((root / "outputs/optimization_report.example.json").read_text(encoding="utf-8")) + inputs = { + "train": root / "data/train.evalset.json", + "validation": root / "data/val.evalset.json", + "optimizer": root / "data/optimizer.json", + "prompt": root / "prompts/baseline_system_prompt.txt", + } + + for name, path in inputs.items(): + assert payload["audit"]["input_hashes"][name] == hashlib.sha256(path.read_bytes()).hexdigest() + serialized = json.dumps(payload, ensure_ascii=False) + assert "C:\\\\Users\\\\" not in serialized + assert "/Users/" not in serialized + assert "/home/" not in serialized +``` + +- [ ] **Step 2: Run the tests and observe overwrite/stale-hash failures** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py -v +``` + +Expected: fake mode deletes/reuses its run directory, duration is zero, and committed example hashes mismatch. + +- [ ] **Step 3: Implement run-specific temporary and final paths** + +In `report.py`, add: + +```python +@dataclass(frozen=True) +class RunArtifactPaths: + output_dir: Path + temporary: Path + final: Path + + +def create_run_artifact_paths(output_dir: str | Path, run_id: str) -> RunArtifactPaths: + root = Path(output_dir) + final = root / "runs" / _safe_artifact_name(run_id) + temporary = root / "runs" / f".{_safe_artifact_name(run_id)}.tmp" + if final.exists() or temporary.exists(): + raise FileExistsError(f"run id already exists: {run_id}") + temporary.mkdir(parents=True) + return RunArtifactPaths(output_dir=root, temporary=temporary, final=final) + + +def finalize_run_directory(paths: RunArtifactPaths) -> None: + paths.temporary.replace(paths.final) + + +def prepare_run_artifacts(report: OptimizationReport, output_dir: str | Path) -> RunArtifactPaths: + paths = create_run_artifact_paths(output_dir, str(report.run["run_id"])) + (paths.temporary / "optimization_report.json").write_text( + report_to_json(report), + encoding="utf-8", + ) + (paths.temporary / "optimization_report.md").write_text( + render_markdown(report), + encoding="utf-8", + ) + write_audit_artifacts(report, paths.temporary) + return paths + + +def _atomic_text(path: Path, value: str) -> None: + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(value, encoding="utf-8") + os.replace(temporary, path) + + +def finalize_run_artifacts(report: OptimizationReport, paths: RunArtifactPaths) -> None: + json_value = report_to_json(report) + markdown_value = render_markdown(report) + _atomic_text(paths.temporary / "optimization_report.json", json_value) + _atomic_text(paths.temporary / "optimization_report.md", markdown_value) + _atomic_text( + paths.temporary / "writeback.json", + json.dumps(to_jsonable(report.writeback), indent=2, sort_keys=True, allow_nan=False) + "\n", + ) + finalize_run_directory(paths) + _atomic_text(paths.output_dir / "optimization_report.json", json_value) + _atomic_text(paths.output_dir / "optimization_report.md", markdown_value) +``` + +Import `os`. Refactor `write_audit_artifacts()` so its second argument is the already-created run directory and it never creates, deletes, or reuses a run ID. Remove the fake-mode `shutil.rmtree(run_dir)` branch. Put SDK optimizer artifacts under the temporary run directory before finalization, not under a shared `/sdk_optimizer` directory. If preparation or finalization raises, do not reach source writeback; leave the temporary directory as failure evidence. + +- [ ] **Step 4: Serialize complete audit artifacts** + +Write baseline and every candidate split under `case_results/`; write `rounds/.json`, `per_case_deltas.json`, `gate_decisions.json`, `writeback.json`, prompt bundles, diffs, input hashes, and config snapshot. Use `allow_nan=False` for every JSON write. + +Normalize displayed paths with: + +```python +def display_path(path: str | Path, *, repository_root: Path) -> str: + resolved = Path(path).resolve() + try: + return resolved.relative_to(repository_root.resolve()).as_posix() + except ValueError: + return Path(path).as_posix() +``` + +Do not serialize the resolved contributor workspace path into committed examples. + +- [ ] **Step 5: Record real duration and keep latest convenience copies** + +Set `audit["duration_seconds"]` from the pipeline `perf_counter` measurement. After the run directory is finalized, atomically replace top-level `optimization_report.json` and `.md` with the finalized report contents. The immutable run directory remains authoritative. + +- [ ] **Step 6: Run reporting tests** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py -v +``` + +Expected: a duplicate run ID raises without modifying the first report, runtime is positive, and all audit JSON is strict. + +- [ ] **Step 7: Commit immutable audit artifacts** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/report.py examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +git commit -m "fix(examples): make optimization audit reproducible" +``` + +### Task 7: Add independent acceptance evidence and real SDK integration + +**Files:** +- Create: `examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json` +- Create: `examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json` +- Create: `examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py` +- Create: `examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py` + +- [ ] **Step 1: Create an independent holdout decision fixture** + +Create `holdout_gate_cases.json` with ten labeled scenarios: + +```json +[ + {"id": "safe_gain", "expected": true, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 0.02, "budget": 1.0}, + {"id": "no_gain", "expected": false, "train": [1, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 0.02, "budget": 1.0}, + {"id": "overfit", "expected": false, "train": [0, 0, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 0, 0], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "validation_regression", "expected": false, "train": [1, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 1, 1], "candidate_validation": [1, 1, 0], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "protected_regression", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 0, 0], "candidate_validation": [0, 1, 1], "protected": ["v0"], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "new_hard_fail", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 0, 0], "candidate_validation": [0, 1, 1], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "soft_to_hard", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.5, 0, 0], "candidate_validation": [0, 1, 1], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "single_case_drop", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.7, 0, 0], "candidate_validation": [0.5, 1, 1], "protected": [], "max_drop": 0.1, "cost": 0.02, "budget": 1.0}, + {"id": "over_budget", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 1.1, "budget": 1.0}, + {"id": "safe_gain_with_soft_failure", "expected": true, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.5, 0, 1], "candidate_validation": [0.5, 1, 1], "protected": ["v0"], "max_drop": 0.0, "cost": 0.02, "budget": 1.0} +] +``` + +Create `attribution_cases.json` with labels separated from evidence: + +```json +[ + {"error_code": "json_parse_failure", "evidence": "JSON parser failed", "expected": "format_violation"}, + {"error_code": "required_key_missing", "evidence": "missing key status", "expected": "final_response_mismatch"}, + {"error_code": "exact_answer_mismatch", "evidence": "expected YES", "expected": "final_response_mismatch"}, + {"error_code": "tool_call_error", "evidence": "expected lookup", "expected": "tool_call_error"}, + {"error_code": "parameter_error", "evidence": "id mismatch", "expected": "parameter_error"}, + {"error_code": "missing_rubric_terms", "evidence": "missing latency", "expected": "llm_rubric_not_met"}, + {"error_code": "knowledge_recall_insufficient", "evidence": "missing doc-a", "expected": "knowledge_recall_insufficient"}, + {"error_code": "forbidden_pattern", "evidence": "forbidden JSON", "expected": "format_violation"} +] +``` + +- [ ] **Step 2: Implement threshold tests** + +In `test_acceptance_thresholds.py`, load fixtures, build `EvalResult` objects without exposing `expected` to the gate or attribution function, and assert: + +```python +def test_holdout_gate_decision_accuracy_is_at_least_eighty_percent(): + scenarios = _load_fixture("holdout_gate_cases.json") + correct = 0 + for scenario in scenarios: + decision = _decision_for_scenario(scenario) + correct += decision.accepted == scenario["expected"] + assert correct / len(scenarios) >= 0.80 + + +def test_independent_attribution_accuracy_is_at_least_seventy_five_percent(): + scenarios = _load_fixture("attribution_cases.json") + predictions = [attribute_failure(item["error_code"], item["evidence"]) for item in scenarios] + correct = sum(prediction[0] == item["expected"] for prediction, item in zip(predictions, scenarios)) + assert correct / len(scenarios) >= 0.75 + assert all(prediction[1] and prediction[2] for prediction in predictions) + + +def test_fake_trace_pipeline_finishes_under_three_minutes(tmp_path: Path): + started = time.perf_counter() + report = run_pipeline(output_dir=tmp_path, mode="fake", trace=True, run_id="performance") + elapsed = time.perf_counter() - started + assert elapsed < 180 + assert 0 < report.audit["duration_seconds"] <= elapsed +``` + +Use these fixture helpers in the same file; only the final accuracy comparison reads the `expected` label: + +```python +FIXTURES = Path(__file__).parent / "fixtures" + + +def _load_fixture(name: str) -> list[dict[str, Any]]: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def _eval(prompt_id: str, split: str, scores: list[float], *, cost: float = 0.0) -> EvalResult: + cases = [ + CaseResult( + case_id=f"{split[0]}{index}", + split=split, + score=float(score), + passed=score >= 1.0, + output=str(score), + metrics={"holdout": float(score)}, + hard_failed=score == 0.0, + ) + for index, score in enumerate(scores) + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=sum(scores) / len(scores), + passed=all(case.passed for case in cases), + cost=cost, + cases=cases, + ) + + +def _decision_for_scenario(scenario: dict[str, Any]) -> GateDecision: + baseline_train = _eval("baseline", "train", scenario["train"]) + baseline_validation = _eval("baseline", "validation", scenario["validation"]) + candidate_train = _eval("candidate", "train", scenario["candidate_train"]) + candidate_validation = _eval("candidate", "validation", scenario["candidate_validation"]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + ) + gate = AcceptanceGate({ + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": scenario["protected"], + "max_score_drop_per_case": scenario["max_drop"], + "max_total_cost": scenario["budget"], + }) + return gate.decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + deltas=deltas, + cumulative_cost=float(scenario["cost"]), + cost_summary=CostSummary(total=float(scenario["cost"]), complete=True), + ) +``` + +Import `Any`, `json`, all referenced schema/gate/report symbols, and `Path`. Here `scenario["cost"]` is supplied as already-incurred cost and the synthetic result costs are zero, matching the production rule that optimizer cost is counted once. + +- [ ] **Step 3: Add a real SDK facade/evaluator integration test** + +In `test_sdk_integration.py`, import the real `AgentOptimizer`, `AgentEvaluator`, `TargetPrompt`, and `GepaReflectiveOptimizer`. Monkeypatch only `GepaReflectiveOptimizer._call_gepa_optimize` with a deterministic GEPA result: + +```python +class FakeGEPAResult: + def __init__(self, baseline: dict[str, str], candidate: dict[str, str]): + self.candidates = [baseline, candidate] + self.val_aggregate_scores = [2 / 3, 1.0] + self.parents = [[None], [0]] + self.discovery_eval_counts = [0, 1] + self.total_metric_calls = 6 + self.best_outputs_valset = None + + @property + def best_idx(self) -> int: + return 1 + + +@pytest.mark.asyncio +async def test_sdk_pipeline_uses_real_facade_evaluator_and_post_gate_writeback(tmp_path: Path, monkeypatch): + prompt_path = tmp_path / "system_prompt.txt" + prompt_path.write_text("baseline prompt", encoding="utf-8") + gate_path = tmp_path / "gate.json" + gate_path.write_text('{"gate": {"max_total_cost": null}}', encoding="utf-8") + baseline = {"system_prompt": "baseline prompt"} + candidate = { + "system_prompt": "baseline prompt\nUse strict JSON only when the user explicitly asks." + } + + async def fake_call_gepa(self, **kwargs): + return FakeGEPAResult(baseline, candidate) + + async def call_agent(query: str) -> str: + prompt = prompt_path.read_text(encoding="utf-8") + return deterministic_sdk_response(prompt, query) + + monkeypatch.setattr(GepaReflectiveOptimizer, "_call_gepa_optimize", fake_call_gepa) + module = ModuleType("issue91_sdk_call_agent") + module.call_agent = call_agent + monkeypatch.setitem(sys.modules, module.__name__, module) + + report = await run_pipeline_async( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + sdk_call_agent="issue91_sdk_call_agent:call_agent", + gate_config_path=gate_path, + update_source=True, + run_id="sdk-integration", + ) + + assert report.selected_candidate is not None + assert all(record["train_result"].cases for record in report.candidates) + assert all(record["validation_result"].cases for record in report.candidates) + assert all(decision.gate_status == "applied" for decision in report.gate_decisions) + assert report.writeback.status == "applied" + assert prompt_path.read_text(encoding="utf-8") == candidate["system_prompt"] +``` + +Define the SDK response helper by routing only the prompt and user query through the oracle-free fake model. The dummy expectation proves the response path does not receive evaluator labels: + +```python +def deterministic_sdk_response(prompt: str, query: str) -> str: + case = EvalCase( + case_id="sdk-runtime-query", + split="runtime", + input=query, + expectation={"type": "runtime-only; must not be inspected"}, + ) + output, _, _ = FakeModel(seed=91).generate("sdk-runtime", prompt, case) + return output +``` + +Import `DEFAULT_TRAIN`, `DEFAULT_VAL`, and `DEFAULT_OPTIMIZER_CONFIG` from `run_pipeline`. The null cost limit is intentional because this deterministic facade test cannot account for external LM billing; the separate gate tests cover configured complete and incomplete costs. Do not patch `AgentEvaluator`, `AgentOptimizer`, or `TargetPrompt`; patch only the external GEPA call shown above. + +- [ ] **Step 4: Run acceptance and SDK integration tests** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py -v +``` + +Expected: gate accuracy is at least 0.80, attribution accuracy is at least 0.75, performance is under 180 seconds, and the real SDK integration produces complete cases and post-gate writeback. + +- [ ] **Step 5: Commit acceptance evidence** + +```powershell +git add examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py +git commit -m "test(examples): prove issue 91 acceptance thresholds" +``` + +### Task 8: Regenerate examples, update documentation, and run merge verification + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/README.md` +- Modify: `examples/optimization/eval_optimize_loop/DESIGN.md` +- Modify: `examples/optimization/eval_optimize_loop/data/optimizer.json` +- Modify: `examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json` +- Modify: `examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md` +- Modify: `.gitignore` +- Modify: `pyproject.toml` + +- [ ] **Step 1: Update user documentation to match the unified semantics** + +Document these exact guarantees in README and the 300–500 字 design summary: + +- fake and SDK share baseline, candidate re-evaluation, delta, and gate semantics; +- SDK optimization always uses `update_source=False` internally; +- `--update-source` writes only the selected candidate after audit preparation and full gate acceptance; +- configured cost gate rejects when total cost is incomplete; +- fake model reads only user input and prompt; +- run directories are immutable and run-specific; +- SDK mode reports real per-case metrics and trace availability. + +Remove every statement describing `partial_applied`, aggregate-only accepted candidates, fixed zero duration, or shared SDK artifact directories. + +- [ ] **Step 2: Regenerate committed example outputs from the final fake pipeline** + +Run: + +```powershell +$output = Join-Path $env:TEMP 'issue91-final-example' +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --trace --run-id example --output-dir $output +Copy-Item -LiteralPath (Join-Path $output 'optimization_report.json') -Destination 'examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json' -Force +Copy-Item -LiteralPath (Join-Path $output 'optimization_report.md') -Destination 'examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md' -Force +``` + +Expected: selected candidate is the safe candidate, the overfit candidate is rejected, duration is positive, input paths are repository-relative, and hashes match current committed inputs. + +- [ ] **Step 3: Run example formatting and focused verification** + +```powershell +python -m compileall -q examples/optimization/eval_optimize_loop +python -m yapf --diff -r examples/optimization/eval_optimize_loop +python -m flake8 examples/optimization/eval_optimize_loop +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +git diff --check +``` + +Expected: every command exits 0, YAPF emits no diff, and the example test suite has zero failures. + +- [ ] **Step 4: Run repository CI-equivalent tests and build** + +```powershell +python -m pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term --cov-fail-under=80 tests/ +python -m build +python -c "import trpc_agent_sdk; print('Import OK')" +``` + +Expected: repository tests pass with coverage at least 80%, package build succeeds, and the import command prints `Import OK`. + +- [ ] **Step 5: Verify the explicit Issue #91 merge checklist** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py -v +rg -n "partial_applied|duration_seconds.: 0\.0|C:\\\\Users\\\\|/Users/|/home/" examples/optimization/eval_optimize_loop +git status --short +git diff --stat origin/main...HEAD +``` + +Expected: threshold tests pass; `rg` finds no stale partial-gate, zero-duration, or personal-path text in runtime/example artifacts; status lists only intentional final changes; diff scope remains limited to Issue #91 and the two superpowers documents. + +- [ ] **Step 6: Commit documentation and generated artifacts** + +```powershell +git add .gitignore pyproject.toml examples/optimization/eval_optimize_loop/README.md examples/optimization/eval_optimize_loop/DESIGN.md examples/optimization/eval_optimize_loop/data/optimizer.json examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md +git commit -m "docs(examples): finalize issue 91 closed loop" +``` + +- [ ] **Step 7: Perform the final branch review before publishing** + +```powershell +git log --oneline origin/main..HEAD +git diff --check origin/main...HEAD +git status --short --branch +``` + +Expected: commits are task-scoped, diff check exits 0, the working tree is clean, and the branch contains no unrelated files. Use `superpowers:requesting-code-review` before push or PR update. From d6d225c5a3a867c77433e37013d31079ea9c3c3b Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 19:19:14 +0800 Subject: [PATCH 12/34] refactor(examples): define safe optimization contracts --- .../eval_optimize_loop/eval_loop/config.py | 62 +++- .../eval_optimize_loop/eval_loop/loader.py | 104 +++++- .../eval_optimize_loop/eval_loop/schemas.py | 75 +++- .../eval_optimize_loop/run_pipeline.py | 330 ++++++++++-------- .../tests/test_config_validation.py | 235 +++++++++++++ .../tests/test_sdk_backend.py | 178 ++++++++-- 6 files changed, 802 insertions(+), 182 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/config.py b/examples/optimization/eval_optimize_loop/eval_loop/config.py index 2bf8fbea..66b37b8e 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/config.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from dataclasses import dataclass from dataclasses import field from pathlib import Path @@ -16,7 +17,7 @@ class GateConfig: allow_new_hard_fail: bool = False protected_case_ids: list[str] = field(default_factory=list) max_score_drop_per_case: float = 0.0 - max_total_cost: float = 1.0 + max_total_cost: float | None = 1.0 extras: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: @@ -125,9 +126,12 @@ def _parse_gate_config(payload: dict[str, Any], *, path: str) -> GateConfig: } extras = {key: value for key, value in payload.items() if key not in allowed} - min_val = payload.get("min_val_score_improvement", 0.01) - if not isinstance(min_val, (int, float)) or min_val < 0 or min_val > 1: - raise ValueError(f"{path}: field 'gate.min_val_score_improvement' must be a number between 0 and 1") + min_val = _finite_number( + payload.get("min_val_score_improvement", 0.01), + f"{path}: field 'gate.min_val_score_improvement'", + 0.0, + 1.0, + ) allow_new_hard_fail = payload.get("allow_new_hard_fail", False) if not isinstance(allow_new_hard_fail, bool): @@ -137,24 +141,56 @@ def _parse_gate_config(payload: dict[str, Any], *, path: str) -> GateConfig: if not isinstance(protected_case_ids, list) or not all(isinstance(item, str) for item in protected_case_ids): raise ValueError(f"{path}: field 'gate.protected_case_ids' must be a list of strings") - max_drop = payload.get("max_score_drop_per_case", 0.0) - if not isinstance(max_drop, (int, float)) or max_drop < 0: - raise ValueError(f"{path}: field 'gate.max_score_drop_per_case' must be a non-negative number") + max_drop = _finite_number( + payload.get("max_score_drop_per_case", 0.0), + f"{path}: field 'gate.max_score_drop_per_case'", + 0.0, + ) - max_total_cost = payload.get("max_total_cost", 1.0) - if not isinstance(max_total_cost, (int, float)) or max_total_cost < 0: - raise ValueError(f"{path}: field 'gate.max_total_cost' must be a non-negative number") + max_total_cost_value = payload.get("max_total_cost", 1.0) + max_total_cost = ( + None + if max_total_cost_value is None + else _finite_number( + max_total_cost_value, + f"{path}: field 'gate.max_total_cost'", + 0.0, + ) + ) return GateConfig( - min_val_score_improvement=float(min_val), + min_val_score_improvement=min_val, allow_new_hard_fail=allow_new_hard_fail, protected_case_ids=list(protected_case_ids), - max_score_drop_per_case=float(max_drop), - max_total_cost=float(max_total_cost), + max_score_drop_per_case=max_drop, + max_total_cost=max_total_cost, extras=extras, ) +def _finite_number( + value: Any, + field_name: str, + minimum: float, + maximum: float | None = None, +) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field_name} must be a finite number") + try: + number = float(value) + except OverflowError as exc: + raise ValueError(f"{field_name} must be a finite number") from exc + if not math.isfinite(number): + raise ValueError(f"{field_name} must be a finite number") + if number < minimum: + raise ValueError(f"{field_name} must be a finite number greater than or equal to {minimum:g}") + if maximum is not None and number > maximum: + raise ValueError( + f"{field_name} must be a finite number between {minimum:g} and {maximum:g}" + ) + return number + + def _validate_cases(cases: list[EvalCase], *, split: str, path: str | Path) -> None: seen: set[str] = set() for case in cases: diff --git a/examples/optimization/eval_optimize_loop/eval_loop/loader.py b/examples/optimization/eval_optimize_loop/eval_loop/loader.py index 29a8628a..7996c67c 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/loader.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/loader.py @@ -13,20 +13,33 @@ def read_json(path: str | Path) -> dict[str, Any]: resolved = Path(path) - with resolved.open("r", encoding="utf-8") as file: - payload = json.load(file) + try: + with resolved.open("r", encoding="utf-8") as file: + payload = json.load(file, parse_constant=_reject_non_standard_json_constant) + except (json.JSONDecodeError, ValueError) as exc: + raise ValueError(f"{resolved}: invalid JSON: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"expected JSON object in {resolved}") return payload +def _reject_non_standard_json_constant(constant: str) -> None: + raise ValueError(f"non-standard JSON constant {constant!r}") + + def load_eval_cases(path: str | Path, split: str | None = None) -> list[EvalCase]: payload = read_json(path) cases = payload.get("cases") - if not isinstance(cases, list): - raise ValueError(f"evalset {path} must contain a cases list") effective_split = split or payload.get("split") or Path(path).name.split(".", 1)[0] - return [EvalCase.from_dict(case, str(effective_split)) for case in cases] + if isinstance(cases, list): + return [EvalCase.from_dict(case, str(effective_split)) for case in cases] + + sdk_cases = payload.get("eval_cases") or payload.get("evalCases") + if isinstance(sdk_cases, list): + _validate_sdk_evalset(payload, path) + return [_eval_case_from_sdk_dict(case, str(effective_split)) for case in sdk_cases] + + raise ValueError(f"evalset {path} must contain a cases or evalCases list") def load_optimizer_config(path: str | Path) -> OptimizerConfig: @@ -53,3 +66,84 @@ def sha256_file(path: str | Path) -> str: for chunk in iter(lambda: file.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() + + +def _validate_sdk_evalset(payload: dict[str, Any], path: str | Path) -> None: + try: + from trpc_agent_sdk.evaluation._eval_set import EvalSet + + EvalSet.model_validate(payload) + except Exception as exc: + raise ValueError(f"evalset {path} is not a valid SDK EvalSet: {exc}") from exc + + +def _eval_case_from_sdk_dict(payload: dict[str, Any], split: str) -> EvalCase: + case_id = payload.get("eval_id") or payload.get("evalId") + if not case_id: + raise ValueError(f"SDK eval case is missing evalId/eval_id: {payload!r}") + + session_input = payload.get("session_input") or payload.get("sessionInput") or {} + state = session_input.get("state") if isinstance(session_input, dict) else {} + state = state if isinstance(state, dict) else {} + expectation = state.get("eval_optimize_expectation") + if not isinstance(expectation, dict): + expectation = _infer_expectation_from_sdk_case(payload) + + return EvalCase( + case_id=str(case_id), + split=split, + input=_first_user_text(payload), + expectation=dict(expectation), + tags=[str(item) for item in state.get("eval_optimize_tags", [])], + protected=bool(state.get("eval_optimize_protected", False)), + simulated_outputs=dict(state.get("eval_optimize_simulated_outputs") or expectation.get("simulated_outputs") or {}), + expected_failure_category=state.get("eval_optimize_expected_failure_category") + or expectation.get("expected_failure_category"), + ) + + +def _infer_expectation_from_sdk_case(payload: dict[str, Any]) -> dict[str, Any]: + expected = _first_final_response_text(payload) + if expected: + return { + "type": "exact", + "expected": expected, + "expected_failure_category": "final_response_mismatch", + } + raise ValueError( + "SDK eval case must put fake-mode metadata in sessionInput.state.eval_optimize_expectation " + f"or provide a finalResponse that can be treated as an exact expectation: {payload!r}" + ) + + +def _first_user_text(payload: dict[str, Any]) -> str: + for invocation in _conversation(payload): + content = invocation.get("user_content") or invocation.get("userContent") or {} + text = _content_text(content) + if text: + return text + return "" + + +def _first_final_response_text(payload: dict[str, Any]) -> str: + for invocation in _conversation(payload): + content = invocation.get("final_response") or invocation.get("finalResponse") or {} + text = _content_text(content) + if text: + return text + return "" + + +def _conversation(payload: dict[str, Any]) -> list[dict[str, Any]]: + conversation = payload.get("conversation") or [] + return [item for item in conversation if isinstance(item, dict)] + + +def _content_text(content: Any) -> str: + if not isinstance(content, dict): + return "" + texts = [] + for part in content.get("parts") or []: + if isinstance(part, dict) and part.get("text") is not None: + texts.append(str(part["text"])) + return "\n".join(texts) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py index c1ce402d..e2f866f4 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -7,6 +7,7 @@ from dataclasses import field from dataclasses import is_dataclass from typing import Any +from typing import Literal @dataclass(frozen=True) @@ -27,12 +28,16 @@ def from_dict(cls, payload: dict[str, Any], split: str) -> "EvalCase": case_id = payload.get("case_id") or payload.get("id") if not case_id: raise ValueError(f"eval case is missing id/case_id: {payload!r}") + if "split" in payload and str(payload["split"]) != str(split): + raise ValueError( + f"eval case {case_id!r} split mismatch: payload has {payload['split']!r}, expected {split!r}" + ) expectation = payload.get("expectation") if not isinstance(expectation, dict): raise ValueError(f"eval case {case_id!r} is missing expectation object") return cls( case_id=str(case_id), - split=str(payload.get("split") or split), + split=str(split), input=str(payload.get("input") or payload.get("user_input") or ""), expectation=dict(expectation), tags=list(payload.get("tags") or []), @@ -52,7 +57,9 @@ class CaseResult: score: float passed: bool output: str + metrics: dict[str, float] = field(default_factory=dict) trace: dict[str, Any] = field(default_factory=dict) + trace_available: bool = False failure_category: str | None = None failure_reason: str | None = None evidence: str | None = None @@ -84,6 +91,67 @@ class CandidatePrompt: prompt: str rationale: str prompt_diff: str + prompt_fields: dict[str, str] = field(default_factory=dict) + + def bundle(self) -> dict[str, str]: + """Return this candidate's complete prompt bundle.""" + + if self.prompt_fields: + return dict(self.prompt_fields) + return {"system_prompt": self.prompt} + + +@dataclass(frozen=True) +class CostSummary: + """Cost attribution for an optimization run.""" + + optimizer: float = 0.0 + evaluator: float = 0.0 + agent: float = 0.0 + total: float = 0.0 + complete: bool = True + + +@dataclass(frozen=True) +class OptimizationRound: + """One auditable optimizer round.""" + + round_id: int + candidate_id: str + prompts: dict[str, str] + rationale: str + metrics: dict[str, float] + cost: CostSummary + duration_seconds: float + + +WritebackStatus = Literal[ + "rejected", + "not_requested", + "applied", + "rolled_back", + "rollback_failed", +] + + +@dataclass(frozen=True) +class WritebackResult: + """Outcome of an optional source prompt writeback.""" + + status: WritebackStatus + before_hashes: dict[str, str] = field(default_factory=dict) + after_hashes: dict[str, str] = field(default_factory=dict) + error: str | None = None + + +@dataclass(frozen=True) +class OptimizationResult: + """Backend-neutral optimization output.""" + + candidates: list[CandidatePrompt] + rounds: list[OptimizationRound] + cost: CostSummary + raw_summary: dict[str, Any] = field(default_factory=dict) @dataclass(frozen=True) @@ -141,6 +209,11 @@ class OptimizationReport: gate_decisions: list[GateDecision] selected_candidate: str | None audit: dict[str, Any] + rounds: list[OptimizationRound] = field(default_factory=list) + cost_summary: CostSummary = field(default_factory=CostSummary) + writeback: WritebackResult = field( + default_factory=lambda: WritebackResult(status="not_requested") + ) def to_jsonable(value: Any) -> Any: diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 9d83ef86..5048584f 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -10,6 +10,7 @@ import shlex import sys import tempfile +from dataclasses import replace from datetime import datetime from datetime import timezone from pathlib import Path @@ -21,8 +22,10 @@ from eval_loop.backends import FakeBackend from eval_loop.backends import SDKBackend +from eval_loop.config import _finite_number from eval_loop.config import validate_inputs from eval_loop.gate import AcceptanceGate +from eval_loop.gate import DEFAULT_GATE_CONFIG from eval_loop.loader import load_eval_cases from eval_loop.loader import load_optimizer_config from eval_loop.loader import load_prompt @@ -345,7 +348,7 @@ def _build_sdk_report( train_case_count: int | None, validation_case_count: int | None, optimizer_config_dict: dict[str, Any], - gate_config: dict[str, float], + gate_config: dict[str, Any], gate_config_path: str | Path | None, target_prompt_paths: dict[str, str | Path], sdk_call_agent: str | None, @@ -369,6 +372,7 @@ def _build_sdk_report( total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0) effective_run_id = run_id or _default_sdk_run_id(sdk_summary) + sdk_eval_config_path = _write_sdk_eval_config(optimizer_config_dict, output_dir) target_prompt_hashes = { name: sha256_file(path) for name, path in target_prompt_paths.items() @@ -378,49 +382,100 @@ def _build_sdk_report( input_hashes["gate_config"] = sha256_file(gate_config_path) availability = { "aggregate_validation_result": True, - "full_train_eval_result": False, - "full_per_case_validation_delta": False, + "full_train_eval_result": True, + "full_per_case_validation_delta": True, } score_explanation = ( - "SDK mode uses OptimizeResult aggregate validation metrics. " - "The full train EvalResult compatibility field is unavailable and keeps score 0.0; " - "full per-case validation deltas are unavailable and listed in not_applied_checks." + "SDK mode uses AgentEvaluator post-optimization runs for train/validation scores, " + "per-case deltas, and gate checks. OptimizeResult aggregate metrics remain in the " + "audit section for cost, duration, token usage, and optimizer round diagnostics." ) - baseline_train = EvalResult(prompt_id="baseline", split="train", score=0.0, passed=False, cost=0.0, cases=[]) - baseline_validation = EvalResult( + baseline_prompts = sdk_backend.last_baseline_prompts or _read_sdk_prompt_dict( + sdk_summary.get("baseline_prompts") + ) + best_prompts = sdk_backend.last_best_prompts or _read_sdk_prompt_dict(sdk_summary.get("best_prompts")) + if not baseline_prompts: + baseline_prompts = {name: Path(path).read_text(encoding="utf-8") for name, path in target_prompt_paths.items()} + if not best_prompts: + best_prompts = {name: candidates[0].prompt for name in target_prompt_paths} if candidates else {} + + eval_output_root = Path(output_dir) / "sdk_evaluator" + baseline_train = sdk_backend.evaluate( prompt_id="baseline", + prompts=baseline_prompts, + eval_dataset_path=train_path, + split="train", + eval_config_path=sdk_eval_config_path, + eval_result_output_dir=eval_output_root / "baseline_train", + ) + baseline_validation = sdk_backend.evaluate( + prompt_id="baseline", + prompts=baseline_prompts, + eval_dataset_path=val_path, split="validation", - score=baseline_pass_rate, - passed=baseline_pass_rate >= 1.0, - cost=0.0, - cases=[], + eval_config_path=sdk_eval_config_path, + eval_result_output_dir=eval_output_root / "baseline_validation", ) - candidate_records = [ - { + + gate_config = _sdk_gate_with_protected_cases(gate_config, val_path) + gate = AcceptanceGate(gate_config) + candidate_records: list[dict[str, Any]] = [] + gate_decisions: list[GateDecision] = [] + all_deltas = [] + cumulative_cost = 0.0 + sdk_status = str(sdk_summary.get("status") or "UNKNOWN") + for candidate in candidates: + candidate_train = sdk_backend.evaluate( + prompt_id=candidate.candidate_id, + prompts=best_prompts, + eval_dataset_path=train_path, + split="train", + eval_config_path=sdk_eval_config_path, + eval_result_output_dir=eval_output_root / f"{candidate.candidate_id}_train", + ) + candidate_validation = sdk_backend.evaluate( + prompt_id=candidate.candidate_id, + prompts=best_prompts, + eval_dataset_path=val_path, + split="validation", + eval_config_path=sdk_eval_config_path, + eval_result_output_dir=eval_output_root / f"{candidate.candidate_id}_validation", + cost=total_llm_cost, + ) + deltas = compute_case_deltas( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + ) + decision = gate.decide( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + deltas=deltas, + cumulative_cost=cumulative_cost, + ) + if sdk_status != "SUCCEEDED": + decision = replace( + decision, + accepted=False, + reasons=[f"reject: SDK optimizer status {sdk_status} is not SUCCEEDED"] + decision.reasons, + ) + cumulative_cost = decision.total_run_cost + all_deltas.extend(deltas) + gate_decisions.append(decision) + candidate_records.append({ "candidate": candidate, - "train_result": EvalResult( - prompt_id=candidate.candidate_id, - split="train", - score=0.0, - passed=False, - cost=0.0, - cases=[], - ), - "validation_result": EvalResult( - prompt_id=candidate.candidate_id, - split="validation", - score=best_pass_rate, - passed=best_pass_rate >= baseline_pass_rate, - cost=total_llm_cost, - cases=[], - ), - "gate_status": "partial_applied", - "gate_not_applied_reason": "SDK OptimizeResult exposes aggregate scores but not full per-case deltas", + "train_result": candidate_train, + "validation_result": candidate_validation, + "gate_status": "applied", "sdk_result_summary": sdk_summary, - } - for candidate in candidates - ] + }) + prompt_hashes = { candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() for candidate in candidates @@ -436,6 +491,7 @@ def _build_sdk_report( "validation": str(val_path), "optimizer": str(optimizer_config_path), "prompt": str(prompt_path), + "sdk_eval_config": str(sdk_eval_config_path) if sdk_eval_config_path else None, }, "prompt_hash": input_hashes["prompt"], "candidate_prompt_hashes": prompt_hashes, @@ -443,13 +499,29 @@ def _build_sdk_report( "target_prompt_hashes": target_prompt_hashes, "sdk_result_availability": availability, "sdk_score_explanation": score_explanation, + "sdk_aggregate_scores": { + "baseline_pass_rate": baseline_pass_rate, + "best_pass_rate": best_pass_rate, + "pass_rate_improvement": pass_rate_improvement, + }, "wrapper_gate_config": dict(gate_config), "wrapper_gate_config_path": str(gate_config_path) if gate_config_path else None, "total_run_cost": total_llm_cost, "cost": { - "baseline": 0.0, - "candidates": {candidate.candidate_id: total_llm_cost for candidate in candidates}, - "total": total_llm_cost, + "baseline": round(baseline_train.cost + baseline_validation.cost, 6), + "candidates": { + record["candidate"].candidate_id: round( + record["train_result"].cost + record["validation_result"].cost, + 6, + ) + for record in candidate_records + }, + "total": round( + baseline_train.cost + + baseline_validation.cost + + sum(record["train_result"].cost + record["validation_result"].cost for record in candidate_records), + 6, + ), }, "candidate_prompts": { candidate.candidate_id: { @@ -497,29 +569,61 @@ def _build_sdk_report( "target_prompts": {name: str(path) for name, path in target_prompt_paths.items()}, "prompt_source": str(prompt_path), } - gate_decisions = [ - _sdk_gate_decision( - candidate_id=candidate.candidate_id, - sdk_summary=sdk_summary, - gate_config=gate_config, - ) - for candidate in candidates - ] selected_candidate = None - if candidates and gate_decisions and gate_decisions[0].accepted: - selected_candidate = candidates[0].candidate_id + for decision in gate_decisions: + if decision.accepted: + selected_candidate = decision.candidate_id + break return build_report( run=run, baseline_train=baseline_train, baseline_validation=baseline_validation, candidate_records=candidate_records, - per_case_deltas=[], + per_case_deltas=all_deltas, gate_decisions=gate_decisions, selected_candidate=selected_candidate, audit=audit, ) +def _write_sdk_eval_config(optimizer_config_dict: dict[str, Any], output_dir: str | Path) -> Path | None: + evaluate_config = optimizer_config_dict.get("evaluate") + if isinstance(evaluate_config, dict): + payload = evaluate_config + else: + payload = { + key: optimizer_config_dict[key] + for key in ("criteria", "metrics", "num_runs", "user_simulator_config") + if key in optimizer_config_dict + } + if not payload: + return None + + path = Path(output_dir) / "sdk_eval_config.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return path + + +def _read_sdk_prompt_dict(value: Any) -> dict[str, str]: + if not isinstance(value, dict): + return {} + return {str(key): str(item) for key, item in value.items() if isinstance(item, str) and item.strip()} + + +def _sdk_gate_with_protected_cases(gate_config: dict[str, Any], val_path: str | Path) -> dict[str, Any]: + merged = dict(gate_config) + if merged.get("protected_case_ids"): + return merged + try: + protected_ids = [case.case_id for case in load_eval_cases(val_path, split="validation") if case.protected] + except Exception: + protected_ids = [] + if protected_ids: + merged["protected_case_ids"] = protected_ids + return merged + + def _sdk_reproducibility_command( *, train_path: str | Path, @@ -563,72 +667,7 @@ def _sdk_reproducibility_command( return " ".join(shlex.quote(part) for part in parts) -def _sdk_gate_decision( - *, - candidate_id: str, - sdk_summary: dict[str, Any], - gate_config: dict[str, float], -) -> GateDecision: - status = str(sdk_summary.get("status") or "UNKNOWN") - improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0, required=True) - total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) - min_improvement = gate_config["min_val_score_improvement"] - max_cost = gate_config["max_total_cost"] - - reasons: list[str] = [] - accepted = True - if status != "SUCCEEDED": - accepted = False - reasons.append(f"reject: SDK optimizer status {status} is not SUCCEEDED") - else: - reasons.append("accept: SDK optimizer status is SUCCEEDED") - - if improvement < min_improvement: - accepted = False - reasons.append( - f"reject: validation improvement {improvement:.3f} is below threshold {min_improvement:.3f}" - ) - else: - reasons.append( - f"accept: validation improvement {improvement:.3f} meets threshold {min_improvement:.3f}" - ) - - if total_cost > max_cost: - accepted = False - reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}") - else: - reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}") - - if accepted: - reasons.append("accept: SDK aggregate gate passed") - - return GateDecision( - candidate_id=candidate_id, - accepted=accepted, - reasons=reasons, - train_score_delta=0.0, - validation_score_delta=improvement, - new_hard_failures=[], - protected_regressions=[], - validation_new_failures=[], - excessive_score_drops=[], - overfit_detected=False, - candidate_cost=total_cost, - cumulative_cost=0.0, - total_run_cost=total_cost, - cost=total_cost, - gate_status="partial_applied", - gate_not_applied_reason="SDK OptimizeResult exposes aggregate scores but not full per-case deltas", - not_applied_checks=[ - "per_case_delta", - "protected_regression", - "new_hard_failure", - "max_score_drop_per_case", - ], - ) - - -def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, float]: +def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, Any]: if gate_config_path is None: gate_payload: dict[str, Any] = {} path_text = "--gate-config" @@ -641,32 +680,46 @@ def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, floa if not isinstance(gate_payload, dict): raise ValueError(f"{path_text}: field 'gate' must be an object when present") - min_improvement = gate_payload.get("min_val_score_improvement", 0.01) - max_cost = gate_payload.get("max_total_cost", 1.0) - if not _is_non_negative_finite_number(min_improvement): - raise ValueError( - f"--gate-config {path_text}: field 'gate.min_val_score_improvement' " - "must be a non-negative finite number" - ) - if not _is_non_negative_finite_number(max_cost): - raise ValueError( - f"--gate-config {path_text}: field 'gate.max_total_cost' must be a non-negative finite number" + merged = dict(DEFAULT_GATE_CONFIG) + merged.update(gate_payload) + min_improvement = merged.get("min_val_score_improvement") + max_cost = merged.get("max_total_cost") + max_drop = merged.get("max_score_drop_per_case") + allow_new_hard_fail = merged.get("allow_new_hard_fail") + protected_case_ids = merged.get("protected_case_ids") + min_improvement = _finite_number( + min_improvement, + f"--gate-config {path_text}: field 'gate.min_val_score_improvement'", + 0.0, + 1.0, + ) + max_drop = _finite_number( + max_drop, + f"--gate-config {path_text}: field 'gate.max_score_drop_per_case'", + 0.0, + ) + if max_cost is not None: + max_cost = _finite_number( + max_cost, + f"--gate-config {path_text}: field 'gate.max_total_cost'", + 0.0, ) + if not isinstance(allow_new_hard_fail, bool): + raise ValueError(f"--gate-config {path_text}: field 'gate.allow_new_hard_fail' must be a boolean") + if not isinstance(protected_case_ids, list) or not all( + isinstance(item, (str, int, float)) and not isinstance(item, bool) + for item in protected_case_ids + ): + raise ValueError(f"--gate-config {path_text}: field 'gate.protected_case_ids' must be a list of ids") return { - "min_val_score_improvement": float(min_improvement), - "max_total_cost": float(max_cost), + "min_val_score_improvement": min_improvement, + "allow_new_hard_fail": bool(allow_new_hard_fail), + "protected_case_ids": [str(item) for item in protected_case_ids], + "max_score_drop_per_case": max_drop, + "max_total_cost": max_cost, } -def _is_non_negative_finite_number(value: Any) -> bool: - return ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and math.isfinite(float(value)) - and float(value) >= 0 - ) - - def _summary_float(summary: dict[str, Any], key: str, default: float, *, required: bool = False) -> float: value = summary.get(key, default) if value is None: @@ -719,6 +772,7 @@ def _parse_target_prompt_paths( if not target_prompts: return {"system_prompt": default_prompt_path} parsed: dict[str, str | Path] = {} + resolved_paths: set[Path] = set() for item in target_prompts: if "=" not in item: raise ValueError("--target-prompt must use name=path format") @@ -732,6 +786,10 @@ def _parse_target_prompt_paths( raise ValueError("--target-prompt must use non-empty name=path values") if name in parsed: raise ValueError(f"--target-prompt duplicate field name {name!r}") + resolved_path = Path(path).resolve() + if resolved_path in resolved_paths: + raise ValueError("--target-prompt fields must not reference the same resolved file") + resolved_paths.add(resolved_path) parsed[name] = Path(path) return parsed diff --git a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py index 1ce4825f..6e5eddc9 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py +++ b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py @@ -5,10 +5,13 @@ import pytest +from examples.optimization.eval_optimize_loop.eval_loop import schemas from examples.optimization.eval_optimize_loop.eval_loop.config import parse_optimizer_config from examples.optimization.eval_optimize_loop.eval_loop.config import validate_inputs from examples.optimization.eval_optimize_loop.eval_loop.loader import load_eval_cases from examples.optimization.eval_optimize_loop.eval_loop.loader import load_optimizer_config +from examples.optimization.eval_optimize_loop.eval_loop.loader import read_json +from examples.optimization.eval_optimize_loop.run_pipeline import _load_sdk_gate_config def test_optimizer_config_defaults_metrics_and_gate(tmp_path: Path): @@ -38,6 +41,238 @@ def test_optimizer_config_rejects_negative_cost(tmp_path: Path): parse_optimizer_config(payload, path=path) +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + (field_name, field_value) + for field_name in ( + "min_val_score_improvement", + "max_score_drop_per_case", + "max_total_cost", + ) + for field_value in (float("nan"), float("inf"), float("-inf"), True) + ], +) +def test_optimizer_config_rejects_non_finite_or_boolean_gate_numbers( + tmp_path: Path, + field_name: str, + field_value: float | bool, +): + path = tmp_path / "optimizer.json" + + with pytest.raises(ValueError, match="finite number"): + parse_optimizer_config({"gate": {field_name: field_value}}, path=path) + + +def test_optimizer_config_allows_disabling_cost_gate(tmp_path: Path): + config = parse_optimizer_config( + {"gate": {"max_total_cost": None}}, + path=tmp_path / "optimizer.json", + ) + + assert config.gate.max_total_cost is None + + +def test_sdk_gate_config_allows_disabling_cost_gate(tmp_path: Path): + path = tmp_path / "gate.json" + path.write_text('{"gate": {"max_total_cost": null}}', encoding="utf-8") + + gate = _load_sdk_gate_config(path) + + assert gate["max_total_cost"] is None + + +@pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity"]) +def test_read_json_rejects_non_standard_constants(tmp_path: Path, constant: str): + path = tmp_path / "invalid.json" + path.write_text(f'{{"value": {constant}}}', encoding="utf-8") + + with pytest.raises(ValueError, match="non-standard JSON constant") as exc_info: + read_json(path) + + assert str(path) in str(exc_info.value) + + +def test_read_json_wraps_decode_errors_with_path(tmp_path: Path): + path = tmp_path / "invalid.json" + path.write_text("{", encoding="utf-8") + + with pytest.raises(ValueError) as exc_info: + read_json(path) + + assert str(path) in str(exc_info.value) + + +def test_eval_case_rejects_explicit_split_mismatch(): + payload = { + "id": "case_1", + "split": "validation", + "input": "hello", + "expectation": {"type": "exact", "expected": "hello"}, + } + + with pytest.raises(ValueError, match="split mismatch"): + schemas.EvalCase.from_dict(payload, split="train") + + +def test_case_result_defaults_metrics_and_trace_availability(): + result = schemas.CaseResult( + case_id="case_1", + split="train", + score=1.0, + passed=True, + output="ok", + ) + + assert result.metrics == {} + assert result.trace == {} + assert result.trace_available is False + + +def test_candidate_prompt_bundle_defaults_to_system_prompt(): + candidate = schemas.CandidatePrompt( + candidate_id="candidate_1", + prompt="system instructions", + rationale="baseline", + prompt_diff="", + ) + + assert candidate.bundle() == {"system_prompt": "system instructions"} + + +def test_candidate_prompt_bundle_returns_prompt_fields_copy(): + prompt_fields = { + "system_prompt": "system instructions", + "router_prompt": "router instructions", + } + candidate = schemas.CandidatePrompt( + candidate_id="candidate_1", + prompt="combined prompt", + rationale="multi-prompt candidate", + prompt_diff="", + prompt_fields=prompt_fields, + ) + + bundle = candidate.bundle() + bundle["system_prompt"] = "changed" + + assert bundle is not prompt_fields + assert candidate.prompt_fields["system_prompt"] == "system instructions" + + +def test_optimization_contract_defaults(): + cost = schemas.CostSummary() + round_result = schemas.OptimizationRound( + round_id=1, + candidate_id="candidate_1", + prompts={"system_prompt": "optimized"}, + rationale="improve format adherence", + metrics={"validation_score": 1.0}, + cost=cost, + duration_seconds=0.25, + ) + result = schemas.OptimizationResult( + candidates=[], + rounds=[round_result], + cost=cost, + ) + + assert cost.optimizer == 0.0 + assert cost.evaluator == 0.0 + assert cost.agent == 0.0 + assert cost.total == 0.0 + assert cost.complete is True + assert result.raw_summary == {} + assert set(schemas.WritebackStatus.__args__) == { + "rejected", + "not_requested", + "applied", + "rolled_back", + "rollback_failed", + } + + +def test_optimization_report_preserves_legacy_construction_with_new_defaults(): + empty_eval = schemas.EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ) + report = schemas.OptimizationReport( + schema_version="1", + run={}, + baseline={}, + baseline_train=empty_eval, + baseline_validation=empty_eval, + candidates=[], + delta={}, + per_case_deltas=[], + failure_attribution_summary={}, + gate_decisions=[], + selected_candidate=None, + audit={}, + ) + + assert report.rounds == [] + assert report.cost_summary == schemas.CostSummary() + assert report.writeback == schemas.WritebackResult(status="not_requested") + + +def test_load_eval_cases_accepts_sdk_evalset_schema(tmp_path: Path): + path = tmp_path / "train.evalset.json" + path.write_text( + json.dumps({ + "evalSetId": "sdk_train", + "evalCases": [ + { + "evalId": "sdk_json_case", + "conversation": [ + { + "invocationId": "turn_1", + "userContent": { + "parts": [{"text": "Return strict JSON with answer=ok."}], + "role": "user", + }, + "finalResponse": { + "parts": [{"text": "{\"answer\":\"ok\"}"}], + "role": "model", + }, + } + ], + "sessionInput": { + "appName": "eval_optimize_loop", + "userId": "tester", + "state": { + "eval_optimize_expectation": { + "type": "json", + "required_keys": ["answer"], + "expected_values": {"answer": "ok"}, + "expected_failure_category": "format_violation", + }, + "eval_optimize_tags": ["json", "hidden"], + "eval_optimize_protected": True, + }, + }, + } + ], + }), + encoding="utf-8", + ) + + cases = load_eval_cases(path, split="train") + + assert [case.case_id for case in cases] == ["sdk_json_case"] + assert cases[0].input == "Return strict JSON with answer=ok." + assert cases[0].expectation["type"] == "json" + assert cases[0].expectation["expected_values"] == {"answer": "ok"} + assert cases[0].tags == ["json", "hidden"] + assert cases[0].protected is True + assert cases[0].expected_failure_category == "format_violation" + + def test_validate_inputs_rejects_same_train_val_path(tmp_path: Path): eval_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) config = parse_optimizer_config({"gate": {}}, path=tmp_path / "optimizer.json") diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index de071c97..a2a75313 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -13,6 +13,7 @@ from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import _parse_target_prompt_paths from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline @@ -302,17 +303,26 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert report.run["mode"] == "sdk" assert report.run["update_source"] is False assert report.selected_candidate == "sdk_best" - assert report.baseline_validation.score == 0.5 - assert report.candidates[0]["validation_result"].score == 0.75 - assert report.gate_decisions[0].validation_score_delta == 0.25 + assert report.baseline_train.score == 0.333333 + assert report.baseline_validation.score == 0.666667 + assert report.candidates[0]["train_result"].score == 1.0 + assert report.candidates[0]["validation_result"].score == 1.0 + assert len(report.per_case_deltas) == 6 + assert { + (delta.split, delta.case_id): delta.delta_type + for delta in report.per_case_deltas + } == { + ("train", "train_json_refund"): "new_pass", + ("train", "train_exact_order_status"): "new_pass", + ("train", "train_rubric_retry_summary"): "unchanged", + ("validation", "val_json_invoice"): "new_pass", + ("validation", "val_explain_cache"): "unchanged", + ("validation", "val_protected_yes_no"): "unchanged", + } + assert report.gate_decisions[0].validation_score_delta == 0.333333 assert report.gate_decisions[0].candidate_cost == 0.123 - assert report.gate_decisions[0].gate_status == "partial_applied" - assert report.gate_decisions[0].not_applied_checks == [ - "per_case_delta", - "protected_regression", - "new_hard_failure", - "max_score_drop_per_case", - ] + assert report.gate_decisions[0].gate_status == "applied" + assert report.gate_decisions[0].not_applied_checks == [] assert report.audit["duration_seconds"] == 12.3 assert report.audit["total_run_cost"] == 0.123 assert report.audit["cost"]["total"] == 0.123 @@ -328,21 +338,28 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert report.audit["sdk_result_summary"]["rounds"][0]["validation_pass_rate"] == 0.75 assert report.audit["sdk_result_availability"] == { "aggregate_validation_result": True, - "full_train_eval_result": False, - "full_per_case_validation_delta": False, + "full_train_eval_result": True, + "full_per_case_validation_delta": True, } - assert "train EvalResult compatibility field is unavailable" in report.audit["sdk_score_explanation"] - assert "partial_applied" in payload - assert "sdk_best (partial_applied)" in markdown - assert "not applied checks: per_case_delta" in markdown - assert "SDK mode uses OptimizeResult aggregate validation metrics" in markdown + assert "AgentEvaluator post-optimization runs" in report.audit["sdk_score_explanation"] + assert "partial_applied" not in payload + assert "sdk_best (accepted)" in markdown + assert "not applied checks" not in markdown + assert "SDK mode uses OptimizeResult aggregate validation metrics" not in markdown assert "fake_call_agent_module:call_agent" in report.run["reproducibility_command"] assert "module:function" not in report.run["reproducibility_command"] assert (output_dir / "runs" / "sdk_test_run" / "input_hashes.json").is_file() assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / "sdk_best.diff").is_file() + assert (output_dir / "runs" / "sdk_test_run" / "case_results" / "sdk_best_validation.json").is_file() assert calls["update_source"] is False assert calls["output_dir"].endswith("sdk_optimizer") assert report.run["sdk_artifact_dir"].endswith("sdk_optimizer") + assert calls["agent_evaluator_runs"] == [ + ("baseline", "train"), + ("baseline", "validation"), + ("sdk_best", "train"), + ("sdk_best", "validation"), + ] def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp_path: Path, monkeypatch): @@ -371,7 +388,8 @@ def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp assert report.run["mode"] == "sdk" assert report.run["train_cases"] == 0 - assert report.selected_candidate == "sdk_best" + assert report.selected_candidate is None + assert report.gate_decisions[0].accepted is False def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path, monkeypatch): @@ -481,14 +499,14 @@ def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_ ) decision = report.gate_decisions[0] - assert report.selected_candidate is None - assert decision.accepted is False - assert decision.gate_status == "partial_applied" - assert decision.validation_score_delta == 0.005 - assert any("validation improvement" in reason for reason in decision.reasons) + assert report.selected_candidate == "sdk_best" + assert decision.accepted is True + assert decision.gate_status == "applied" + assert decision.validation_score_delta == 0.333333 + assert any("validation score improved" in reason for reason in decision.reasons) -def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_improvement( +def test_run_pipeline_mode_sdk_custom_gate_rejects_low_post_eval_validation_improvement( tmp_path: Path, monkeypatch, ): @@ -500,7 +518,7 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_impr pass_rate_improvement=0.25, total_llm_cost=0.123, ) - gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.3, max_total_cost=1.0) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.5, max_total_cost=1.0) report = run_pipeline( mode="sdk", @@ -516,7 +534,7 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_impr decision = report.gate_decisions[0] assert report.selected_candidate is None assert decision.accepted is False - assert decision.validation_score_delta == 0.25 + assert decision.validation_score_delta == 0.333333 assert any("validation improvement" in reason for reason in decision.reasons) @@ -545,7 +563,7 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Pa decision = report.gate_decisions[0] assert report.selected_candidate is None assert decision.accepted is False - assert decision.gate_status == "partial_applied" + assert decision.gate_status == "applied" assert decision.total_run_cost == 2.0 assert any("cost" in reason for reason in decision.reasons) @@ -628,6 +646,21 @@ def test_run_pipeline_mode_sdk_rejects_invalid_target_prompt_field_names( assert repr(field_name) in str(exc_info.value) +def test_target_prompt_paths_reject_same_resolved_file(tmp_path: Path): + prompt_path = tmp_path / "prompt.txt" + equivalent_path = tmp_path / "nested" / ".." / prompt_path.name + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="same resolved file"): + _parse_target_prompt_paths( + [ + f"system_prompt={prompt_path}", + f"router_prompt={equivalent_path}", + ], + default_prompt_path=prompt_path, + ) + + @pytest.mark.parametrize( ("field_name", "field_value"), [ @@ -846,8 +879,40 @@ async def optimize(**kwargs): ], ) + class FakeAgentEvaluator: + @staticmethod + def get_executer(eval_dataset_file_path_or_dir, **kwargs): + return FakeEvalExecuter(Path(eval_dataset_file_path_or_dir)) + + class FakeEvalExecuter: + def __init__(self, eval_path: Path): + self.eval_path = eval_path + self._result = None + + async def evaluate(self): + split = "train" if "train" in self.eval_path.name else "validation" + prompt_label = _current_prompt_label(calls) + calls.setdefault("agent_evaluator_runs", []).append((prompt_label, split)) + scores = _fake_eval_scores(self.eval_path, split=split, prompt_label=prompt_label) + self._result = types.SimpleNamespace( + results_by_eval_set_id={ + f"{split}_set": types.SimpleNamespace( + eval_results_by_eval_id={ + case_id: [_fake_case_result(case_id, score)] + for case_id, score in scores + } + ) + } + ) + if any(score < 1.0 for _, score in scores): + raise AssertionError("evaluation cases failed") + + def get_result(self): + return self._result + fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") fake_eval_module.AgentOptimizer = FakeAgentOptimizer + fake_eval_module.AgentEvaluator = FakeAgentEvaluator fake_eval_module.TargetPrompt = FakeTargetPrompt monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) @@ -861,6 +926,65 @@ async def call_agent(query: str) -> str: return calls +def _current_prompt_label(calls: dict) -> str: + target_prompt = calls.get("target_prompt") + paths = getattr(target_prompt, "paths", []) if target_prompt is not None else [] + contents = [ + Path(path).read_text(encoding="utf-8") + for _, path in paths + if Path(path).is_file() + ] + return "sdk_best" if any("optimized" in content for content in contents) else "baseline" + + +def _fake_eval_scores(eval_path: Path, *, split: str, prompt_label: str) -> list[tuple[str, float]]: + case_ids = _case_ids(eval_path) + if not case_ids: + return [] + if prompt_label == "sdk_best": + return [(case_id, 1.0) for case_id in case_ids] + baseline_scores = { + "train_json_refund": 0.0, + "train_exact_order_status": 0.0, + "train_rubric_retry_summary": 1.0, + "val_json_invoice": 0.0, + "val_explain_cache": 1.0, + "val_protected_yes_no": 1.0, + } + return [(case_id, baseline_scores.get(case_id, 0.5 if split == "validation" else 0.0)) for case_id in case_ids] + + +def _case_ids(eval_path: Path) -> list[str]: + payload = json.loads(eval_path.read_text(encoding="utf-8")) + cases = payload.get("cases") or payload.get("eval_cases") or payload.get("evalCases") or [] + ids = [] + for case in cases: + if isinstance(case, dict): + case_id = case.get("id") or case.get("case_id") or case.get("eval_id") or case.get("evalId") + if case_id: + ids.append(str(case_id)) + return ids + + +def _fake_case_result(case_id: str, score: float): + passed = score >= 1.0 + status = "PASSED" if passed else "FAILED" + return types.SimpleNamespace( + eval_id=case_id, + final_eval_status=status, + error_message=None if passed else "response did not match expectation", + overall_eval_metric_results=[ + types.SimpleNamespace( + metric_name="response_match_score", + score=score, + eval_status=status, + details=types.SimpleNamespace(reason="response did not match expectation"), + ) + ], + eval_metric_result_per_invocation=[], + ) + + def _write_sdk_optimizer_config(tmp_path: Path) -> Path: path = tmp_path / "sdk_optimizer.json" path.write_text( From dae8692bd6b1212e4ae7735241d193a56f9d3845 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 19:30:55 +0800 Subject: [PATCH 13/34] fix(examples): keep contract change self-contained --- .../eval_optimize_loop/eval_loop/loader.py | 93 +----- .../eval_optimize_loop/eval_loop/schemas.py | 4 +- .../eval_optimize_loop/run_pipeline.py | 288 +++++++----------- .../tests/test_config_validation.py | 83 ++--- .../tests/test_sdk_backend.py | 166 ++-------- 5 files changed, 181 insertions(+), 453 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/loader.py b/examples/optimization/eval_optimize_loop/eval_loop/loader.py index 7996c67c..62659a16 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/loader.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/loader.py @@ -30,16 +30,10 @@ def _reject_non_standard_json_constant(constant: str) -> None: def load_eval_cases(path: str | Path, split: str | None = None) -> list[EvalCase]: payload = read_json(path) cases = payload.get("cases") + if not isinstance(cases, list): + raise ValueError(f"evalset {path} must contain a cases list") effective_split = split or payload.get("split") or Path(path).name.split(".", 1)[0] - if isinstance(cases, list): - return [EvalCase.from_dict(case, str(effective_split)) for case in cases] - - sdk_cases = payload.get("eval_cases") or payload.get("evalCases") - if isinstance(sdk_cases, list): - _validate_sdk_evalset(payload, path) - return [_eval_case_from_sdk_dict(case, str(effective_split)) for case in sdk_cases] - - raise ValueError(f"evalset {path} must contain a cases or evalCases list") + return [EvalCase.from_dict(case, str(effective_split)) for case in cases] def load_optimizer_config(path: str | Path) -> OptimizerConfig: @@ -66,84 +60,3 @@ def sha256_file(path: str | Path) -> str: for chunk in iter(lambda: file.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() - - -def _validate_sdk_evalset(payload: dict[str, Any], path: str | Path) -> None: - try: - from trpc_agent_sdk.evaluation._eval_set import EvalSet - - EvalSet.model_validate(payload) - except Exception as exc: - raise ValueError(f"evalset {path} is not a valid SDK EvalSet: {exc}") from exc - - -def _eval_case_from_sdk_dict(payload: dict[str, Any], split: str) -> EvalCase: - case_id = payload.get("eval_id") or payload.get("evalId") - if not case_id: - raise ValueError(f"SDK eval case is missing evalId/eval_id: {payload!r}") - - session_input = payload.get("session_input") or payload.get("sessionInput") or {} - state = session_input.get("state") if isinstance(session_input, dict) else {} - state = state if isinstance(state, dict) else {} - expectation = state.get("eval_optimize_expectation") - if not isinstance(expectation, dict): - expectation = _infer_expectation_from_sdk_case(payload) - - return EvalCase( - case_id=str(case_id), - split=split, - input=_first_user_text(payload), - expectation=dict(expectation), - tags=[str(item) for item in state.get("eval_optimize_tags", [])], - protected=bool(state.get("eval_optimize_protected", False)), - simulated_outputs=dict(state.get("eval_optimize_simulated_outputs") or expectation.get("simulated_outputs") or {}), - expected_failure_category=state.get("eval_optimize_expected_failure_category") - or expectation.get("expected_failure_category"), - ) - - -def _infer_expectation_from_sdk_case(payload: dict[str, Any]) -> dict[str, Any]: - expected = _first_final_response_text(payload) - if expected: - return { - "type": "exact", - "expected": expected, - "expected_failure_category": "final_response_mismatch", - } - raise ValueError( - "SDK eval case must put fake-mode metadata in sessionInput.state.eval_optimize_expectation " - f"or provide a finalResponse that can be treated as an exact expectation: {payload!r}" - ) - - -def _first_user_text(payload: dict[str, Any]) -> str: - for invocation in _conversation(payload): - content = invocation.get("user_content") or invocation.get("userContent") or {} - text = _content_text(content) - if text: - return text - return "" - - -def _first_final_response_text(payload: dict[str, Any]) -> str: - for invocation in _conversation(payload): - content = invocation.get("final_response") or invocation.get("finalResponse") or {} - text = _content_text(content) - if text: - return text - return "" - - -def _conversation(payload: dict[str, Any]) -> list[dict[str, Any]]: - conversation = payload.get("conversation") or [] - return [item for item in conversation if isinstance(item, dict)] - - -def _content_text(content: Any) -> str: - if not isinstance(content, dict): - return "" - texts = [] - for part in content.get("parts") or []: - if isinstance(part, dict) and part.get("text") is not None: - texts.append(str(part["text"])) - return "\n".join(texts) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py index e2f866f4..04da133e 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -57,9 +57,9 @@ class CaseResult: score: float passed: bool output: str - metrics: dict[str, float] = field(default_factory=dict) + metrics: dict[str, float] = field(default_factory=dict, kw_only=True) trace: dict[str, Any] = field(default_factory=dict) - trace_available: bool = False + trace_available: bool = field(default=False, kw_only=True) failure_category: str | None = None failure_reason: str | None = None evidence: str | None = None diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 5048584f..90863840 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -10,7 +10,6 @@ import shlex import sys import tempfile -from dataclasses import replace from datetime import datetime from datetime import timezone from pathlib import Path @@ -25,7 +24,6 @@ from eval_loop.config import _finite_number from eval_loop.config import validate_inputs from eval_loop.gate import AcceptanceGate -from eval_loop.gate import DEFAULT_GATE_CONFIG from eval_loop.loader import load_eval_cases from eval_loop.loader import load_optimizer_config from eval_loop.loader import load_prompt @@ -372,7 +370,6 @@ def _build_sdk_report( total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0) effective_run_id = run_id or _default_sdk_run_id(sdk_summary) - sdk_eval_config_path = _write_sdk_eval_config(optimizer_config_dict, output_dir) target_prompt_hashes = { name: sha256_file(path) for name, path in target_prompt_paths.items() @@ -382,100 +379,49 @@ def _build_sdk_report( input_hashes["gate_config"] = sha256_file(gate_config_path) availability = { "aggregate_validation_result": True, - "full_train_eval_result": True, - "full_per_case_validation_delta": True, + "full_train_eval_result": False, + "full_per_case_validation_delta": False, } score_explanation = ( - "SDK mode uses AgentEvaluator post-optimization runs for train/validation scores, " - "per-case deltas, and gate checks. OptimizeResult aggregate metrics remain in the " - "audit section for cost, duration, token usage, and optimizer round diagnostics." + "SDK mode uses OptimizeResult aggregate validation metrics. " + "The full train EvalResult compatibility field is unavailable and keeps score 0.0; " + "full per-case validation deltas are unavailable and listed in not_applied_checks." ) - baseline_prompts = sdk_backend.last_baseline_prompts or _read_sdk_prompt_dict( - sdk_summary.get("baseline_prompts") - ) - best_prompts = sdk_backend.last_best_prompts or _read_sdk_prompt_dict(sdk_summary.get("best_prompts")) - if not baseline_prompts: - baseline_prompts = {name: Path(path).read_text(encoding="utf-8") for name, path in target_prompt_paths.items()} - if not best_prompts: - best_prompts = {name: candidates[0].prompt for name in target_prompt_paths} if candidates else {} - - eval_output_root = Path(output_dir) / "sdk_evaluator" - baseline_train = sdk_backend.evaluate( + baseline_train = EvalResult(prompt_id="baseline", split="train", score=0.0, passed=False, cost=0.0, cases=[]) + baseline_validation = EvalResult( prompt_id="baseline", - prompts=baseline_prompts, - eval_dataset_path=train_path, - split="train", - eval_config_path=sdk_eval_config_path, - eval_result_output_dir=eval_output_root / "baseline_train", - ) - baseline_validation = sdk_backend.evaluate( - prompt_id="baseline", - prompts=baseline_prompts, - eval_dataset_path=val_path, split="validation", - eval_config_path=sdk_eval_config_path, - eval_result_output_dir=eval_output_root / "baseline_validation", + score=baseline_pass_rate, + passed=baseline_pass_rate >= 1.0, + cost=0.0, + cases=[], ) - - gate_config = _sdk_gate_with_protected_cases(gate_config, val_path) - gate = AcceptanceGate(gate_config) - candidate_records: list[dict[str, Any]] = [] - gate_decisions: list[GateDecision] = [] - all_deltas = [] - cumulative_cost = 0.0 - sdk_status = str(sdk_summary.get("status") or "UNKNOWN") - for candidate in candidates: - candidate_train = sdk_backend.evaluate( - prompt_id=candidate.candidate_id, - prompts=best_prompts, - eval_dataset_path=train_path, - split="train", - eval_config_path=sdk_eval_config_path, - eval_result_output_dir=eval_output_root / f"{candidate.candidate_id}_train", - ) - candidate_validation = sdk_backend.evaluate( - prompt_id=candidate.candidate_id, - prompts=best_prompts, - eval_dataset_path=val_path, - split="validation", - eval_config_path=sdk_eval_config_path, - eval_result_output_dir=eval_output_root / f"{candidate.candidate_id}_validation", - cost=total_llm_cost, - ) - deltas = compute_case_deltas( - candidate_id=candidate.candidate_id, - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_train=candidate_train, - candidate_validation=candidate_validation, - ) - decision = gate.decide( - candidate_id=candidate.candidate_id, - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_train=candidate_train, - candidate_validation=candidate_validation, - deltas=deltas, - cumulative_cost=cumulative_cost, - ) - if sdk_status != "SUCCEEDED": - decision = replace( - decision, - accepted=False, - reasons=[f"reject: SDK optimizer status {sdk_status} is not SUCCEEDED"] + decision.reasons, - ) - cumulative_cost = decision.total_run_cost - all_deltas.extend(deltas) - gate_decisions.append(decision) - candidate_records.append({ + candidate_records = [ + { "candidate": candidate, - "train_result": candidate_train, - "validation_result": candidate_validation, - "gate_status": "applied", + "train_result": EvalResult( + prompt_id=candidate.candidate_id, + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ), + "validation_result": EvalResult( + prompt_id=candidate.candidate_id, + split="validation", + score=best_pass_rate, + passed=best_pass_rate >= baseline_pass_rate, + cost=total_llm_cost, + cases=[], + ), + "gate_status": "partial_applied", + "gate_not_applied_reason": "SDK OptimizeResult exposes aggregate scores but not full per-case deltas", "sdk_result_summary": sdk_summary, - }) - + } + for candidate in candidates + ] prompt_hashes = { candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() for candidate in candidates @@ -491,7 +437,6 @@ def _build_sdk_report( "validation": str(val_path), "optimizer": str(optimizer_config_path), "prompt": str(prompt_path), - "sdk_eval_config": str(sdk_eval_config_path) if sdk_eval_config_path else None, }, "prompt_hash": input_hashes["prompt"], "candidate_prompt_hashes": prompt_hashes, @@ -499,29 +444,13 @@ def _build_sdk_report( "target_prompt_hashes": target_prompt_hashes, "sdk_result_availability": availability, "sdk_score_explanation": score_explanation, - "sdk_aggregate_scores": { - "baseline_pass_rate": baseline_pass_rate, - "best_pass_rate": best_pass_rate, - "pass_rate_improvement": pass_rate_improvement, - }, "wrapper_gate_config": dict(gate_config), "wrapper_gate_config_path": str(gate_config_path) if gate_config_path else None, "total_run_cost": total_llm_cost, "cost": { - "baseline": round(baseline_train.cost + baseline_validation.cost, 6), - "candidates": { - record["candidate"].candidate_id: round( - record["train_result"].cost + record["validation_result"].cost, - 6, - ) - for record in candidate_records - }, - "total": round( - baseline_train.cost - + baseline_validation.cost - + sum(record["train_result"].cost + record["validation_result"].cost for record in candidate_records), - 6, - ), + "baseline": 0.0, + "candidates": {candidate.candidate_id: total_llm_cost for candidate in candidates}, + "total": total_llm_cost, }, "candidate_prompts": { candidate.candidate_id: { @@ -569,61 +498,29 @@ def _build_sdk_report( "target_prompts": {name: str(path) for name, path in target_prompt_paths.items()}, "prompt_source": str(prompt_path), } + gate_decisions = [ + _sdk_gate_decision( + candidate_id=candidate.candidate_id, + sdk_summary=sdk_summary, + gate_config=gate_config, + ) + for candidate in candidates + ] selected_candidate = None - for decision in gate_decisions: - if decision.accepted: - selected_candidate = decision.candidate_id - break + if candidates and gate_decisions and gate_decisions[0].accepted: + selected_candidate = candidates[0].candidate_id return build_report( run=run, baseline_train=baseline_train, baseline_validation=baseline_validation, candidate_records=candidate_records, - per_case_deltas=all_deltas, + per_case_deltas=[], gate_decisions=gate_decisions, selected_candidate=selected_candidate, audit=audit, ) -def _write_sdk_eval_config(optimizer_config_dict: dict[str, Any], output_dir: str | Path) -> Path | None: - evaluate_config = optimizer_config_dict.get("evaluate") - if isinstance(evaluate_config, dict): - payload = evaluate_config - else: - payload = { - key: optimizer_config_dict[key] - for key in ("criteria", "metrics", "num_runs", "user_simulator_config") - if key in optimizer_config_dict - } - if not payload: - return None - - path = Path(output_dir) / "sdk_eval_config.json" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - return path - - -def _read_sdk_prompt_dict(value: Any) -> dict[str, str]: - if not isinstance(value, dict): - return {} - return {str(key): str(item) for key, item in value.items() if isinstance(item, str) and item.strip()} - - -def _sdk_gate_with_protected_cases(gate_config: dict[str, Any], val_path: str | Path) -> dict[str, Any]: - merged = dict(gate_config) - if merged.get("protected_case_ids"): - return merged - try: - protected_ids = [case.case_id for case in load_eval_cases(val_path, split="validation") if case.protected] - except Exception: - protected_ids = [] - if protected_ids: - merged["protected_case_ids"] = protected_ids - return merged - - def _sdk_reproducibility_command( *, train_path: str | Path, @@ -667,6 +564,71 @@ def _sdk_reproducibility_command( return " ".join(shlex.quote(part) for part in parts) +def _sdk_gate_decision( + *, + candidate_id: str, + sdk_summary: dict[str, Any], + gate_config: dict[str, Any], +) -> GateDecision: + status = str(sdk_summary.get("status") or "UNKNOWN") + improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0, required=True) + total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) + min_improvement = gate_config["min_val_score_improvement"] + max_cost = gate_config["max_total_cost"] + + reasons: list[str] = [] + accepted = True + if status != "SUCCEEDED": + accepted = False + reasons.append(f"reject: SDK optimizer status {status} is not SUCCEEDED") + else: + reasons.append("accept: SDK optimizer status is SUCCEEDED") + + if improvement < min_improvement: + accepted = False + reasons.append( + f"reject: validation improvement {improvement:.3f} is below threshold {min_improvement:.3f}" + ) + else: + reasons.append( + f"accept: validation improvement {improvement:.3f} meets threshold {min_improvement:.3f}" + ) + + if total_cost > max_cost: + accepted = False + reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}") + else: + reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}") + + if accepted: + reasons.append("accept: SDK aggregate gate passed") + + return GateDecision( + candidate_id=candidate_id, + accepted=accepted, + reasons=reasons, + train_score_delta=0.0, + validation_score_delta=improvement, + new_hard_failures=[], + protected_regressions=[], + validation_new_failures=[], + excessive_score_drops=[], + overfit_detected=False, + candidate_cost=total_cost, + cumulative_cost=0.0, + total_run_cost=total_cost, + cost=total_cost, + gate_status="partial_applied", + gate_not_applied_reason="SDK OptimizeResult exposes aggregate scores but not full per-case deltas", + not_applied_checks=[ + "per_case_delta", + "protected_regression", + "new_hard_failure", + "max_score_drop_per_case", + ], + ) + + def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, Any]: if gate_config_path is None: gate_payload: dict[str, Any] = {} @@ -680,42 +642,22 @@ def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, Any] if not isinstance(gate_payload, dict): raise ValueError(f"{path_text}: field 'gate' must be an object when present") - merged = dict(DEFAULT_GATE_CONFIG) - merged.update(gate_payload) - min_improvement = merged.get("min_val_score_improvement") - max_cost = merged.get("max_total_cost") - max_drop = merged.get("max_score_drop_per_case") - allow_new_hard_fail = merged.get("allow_new_hard_fail") - protected_case_ids = merged.get("protected_case_ids") + min_improvement = gate_payload.get("min_val_score_improvement", 0.01) + max_cost = gate_payload.get("max_total_cost", 1.0) min_improvement = _finite_number( min_improvement, f"--gate-config {path_text}: field 'gate.min_val_score_improvement'", 0.0, 1.0, ) - max_drop = _finite_number( - max_drop, - f"--gate-config {path_text}: field 'gate.max_score_drop_per_case'", - 0.0, - ) if max_cost is not None: max_cost = _finite_number( max_cost, f"--gate-config {path_text}: field 'gate.max_total_cost'", 0.0, ) - if not isinstance(allow_new_hard_fail, bool): - raise ValueError(f"--gate-config {path_text}: field 'gate.allow_new_hard_fail' must be a boolean") - if not isinstance(protected_case_ids, list) or not all( - isinstance(item, (str, int, float)) and not isinstance(item, bool) - for item in protected_case_ids - ): - raise ValueError(f"--gate-config {path_text}: field 'gate.protected_case_ids' must be a list of ids") return { "min_val_score_improvement": min_improvement, - "allow_new_hard_fail": bool(allow_new_hard_fail), - "protected_case_ids": [str(item) for item in protected_case_ids], - "max_score_drop_per_case": max_drop, "max_total_cost": max_cost, } diff --git a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py index 6e5eddc9..18c5cafe 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py +++ b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py @@ -129,6 +129,37 @@ def test_case_result_defaults_metrics_and_trace_availability(): assert result.trace_available is False +def test_case_result_preserves_legacy_positional_arguments_with_new_keyword_fields(): + trace = {"events": ["model_call"]} + + result = schemas.CaseResult( + "case_1", + "validation", + 0.25, + False, + "bad output", + trace, + "format_violation", + "invalid JSON", + "parser rejected output", + 0.125, + True, + "format_violation", + metrics={"response_match": 0.25}, + trace_available=True, + ) + + assert result.trace == trace + assert result.failure_category == "format_violation" + assert result.failure_reason == "invalid JSON" + assert result.evidence == "parser rejected output" + assert result.cost == 0.125 + assert result.hard_failed is True + assert result.expected_failure_category == "format_violation" + assert result.metrics == {"response_match": 0.25} + assert result.trace_available is True + + def test_candidate_prompt_bundle_defaults_to_system_prompt(): candidate = schemas.CandidatePrompt( candidate_id="candidate_1", @@ -221,58 +252,6 @@ def test_optimization_report_preserves_legacy_construction_with_new_defaults(): assert report.writeback == schemas.WritebackResult(status="not_requested") -def test_load_eval_cases_accepts_sdk_evalset_schema(tmp_path: Path): - path = tmp_path / "train.evalset.json" - path.write_text( - json.dumps({ - "evalSetId": "sdk_train", - "evalCases": [ - { - "evalId": "sdk_json_case", - "conversation": [ - { - "invocationId": "turn_1", - "userContent": { - "parts": [{"text": "Return strict JSON with answer=ok."}], - "role": "user", - }, - "finalResponse": { - "parts": [{"text": "{\"answer\":\"ok\"}"}], - "role": "model", - }, - } - ], - "sessionInput": { - "appName": "eval_optimize_loop", - "userId": "tester", - "state": { - "eval_optimize_expectation": { - "type": "json", - "required_keys": ["answer"], - "expected_values": {"answer": "ok"}, - "expected_failure_category": "format_violation", - }, - "eval_optimize_tags": ["json", "hidden"], - "eval_optimize_protected": True, - }, - }, - } - ], - }), - encoding="utf-8", - ) - - cases = load_eval_cases(path, split="train") - - assert [case.case_id for case in cases] == ["sdk_json_case"] - assert cases[0].input == "Return strict JSON with answer=ok." - assert cases[0].expectation["type"] == "json" - assert cases[0].expectation["expected_values"] == {"answer": "ok"} - assert cases[0].tags == ["json", "hidden"] - assert cases[0].protected is True - assert cases[0].expected_failure_category == "format_violation" - - def test_validate_inputs_rejects_same_train_val_path(tmp_path: Path): eval_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) config = parse_optimizer_config({"gate": {}}, path=tmp_path / "optimizer.json") diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index a2a75313..d89dd4ee 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -303,26 +303,17 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert report.run["mode"] == "sdk" assert report.run["update_source"] is False assert report.selected_candidate == "sdk_best" - assert report.baseline_train.score == 0.333333 - assert report.baseline_validation.score == 0.666667 - assert report.candidates[0]["train_result"].score == 1.0 - assert report.candidates[0]["validation_result"].score == 1.0 - assert len(report.per_case_deltas) == 6 - assert { - (delta.split, delta.case_id): delta.delta_type - for delta in report.per_case_deltas - } == { - ("train", "train_json_refund"): "new_pass", - ("train", "train_exact_order_status"): "new_pass", - ("train", "train_rubric_retry_summary"): "unchanged", - ("validation", "val_json_invoice"): "new_pass", - ("validation", "val_explain_cache"): "unchanged", - ("validation", "val_protected_yes_no"): "unchanged", - } - assert report.gate_decisions[0].validation_score_delta == 0.333333 + assert report.baseline_validation.score == 0.5 + assert report.candidates[0]["validation_result"].score == 0.75 + assert report.gate_decisions[0].validation_score_delta == 0.25 assert report.gate_decisions[0].candidate_cost == 0.123 - assert report.gate_decisions[0].gate_status == "applied" - assert report.gate_decisions[0].not_applied_checks == [] + assert report.gate_decisions[0].gate_status == "partial_applied" + assert report.gate_decisions[0].not_applied_checks == [ + "per_case_delta", + "protected_regression", + "new_hard_failure", + "max_score_drop_per_case", + ] assert report.audit["duration_seconds"] == 12.3 assert report.audit["total_run_cost"] == 0.123 assert report.audit["cost"]["total"] == 0.123 @@ -338,28 +329,21 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert report.audit["sdk_result_summary"]["rounds"][0]["validation_pass_rate"] == 0.75 assert report.audit["sdk_result_availability"] == { "aggregate_validation_result": True, - "full_train_eval_result": True, - "full_per_case_validation_delta": True, + "full_train_eval_result": False, + "full_per_case_validation_delta": False, } - assert "AgentEvaluator post-optimization runs" in report.audit["sdk_score_explanation"] - assert "partial_applied" not in payload - assert "sdk_best (accepted)" in markdown - assert "not applied checks" not in markdown - assert "SDK mode uses OptimizeResult aggregate validation metrics" not in markdown + assert "train EvalResult compatibility field is unavailable" in report.audit["sdk_score_explanation"] + assert "partial_applied" in payload + assert "sdk_best (partial_applied)" in markdown + assert "not applied checks: per_case_delta" in markdown + assert "SDK mode uses OptimizeResult aggregate validation metrics" in markdown assert "fake_call_agent_module:call_agent" in report.run["reproducibility_command"] assert "module:function" not in report.run["reproducibility_command"] assert (output_dir / "runs" / "sdk_test_run" / "input_hashes.json").is_file() assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / "sdk_best.diff").is_file() - assert (output_dir / "runs" / "sdk_test_run" / "case_results" / "sdk_best_validation.json").is_file() assert calls["update_source"] is False assert calls["output_dir"].endswith("sdk_optimizer") assert report.run["sdk_artifact_dir"].endswith("sdk_optimizer") - assert calls["agent_evaluator_runs"] == [ - ("baseline", "train"), - ("baseline", "validation"), - ("sdk_best", "train"), - ("sdk_best", "validation"), - ] def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp_path: Path, monkeypatch): @@ -388,8 +372,7 @@ def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp assert report.run["mode"] == "sdk" assert report.run["train_cases"] == 0 - assert report.selected_candidate is None - assert report.gate_decisions[0].accepted is False + assert report.selected_candidate == "sdk_best" def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path, monkeypatch): @@ -499,14 +482,14 @@ def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_ ) decision = report.gate_decisions[0] - assert report.selected_candidate == "sdk_best" - assert decision.accepted is True - assert decision.gate_status == "applied" - assert decision.validation_score_delta == 0.333333 - assert any("validation score improved" in reason for reason in decision.reasons) + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.gate_status == "partial_applied" + assert decision.validation_score_delta == 0.005 + assert any("validation improvement" in reason for reason in decision.reasons) -def test_run_pipeline_mode_sdk_custom_gate_rejects_low_post_eval_validation_improvement( +def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_improvement( tmp_path: Path, monkeypatch, ): @@ -518,7 +501,7 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_low_post_eval_validation_impr pass_rate_improvement=0.25, total_llm_cost=0.123, ) - gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.5, max_total_cost=1.0) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.3, max_total_cost=1.0) report = run_pipeline( mode="sdk", @@ -534,7 +517,7 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_low_post_eval_validation_impr decision = report.gate_decisions[0] assert report.selected_candidate is None assert decision.accepted is False - assert decision.validation_score_delta == 0.333333 + assert decision.validation_score_delta == 0.25 assert any("validation improvement" in reason for reason in decision.reasons) @@ -563,7 +546,7 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Pa decision = report.gate_decisions[0] assert report.selected_candidate is None assert decision.accepted is False - assert decision.gate_status == "applied" + assert decision.gate_status == "partial_applied" assert decision.total_run_cost == 2.0 assert any("cost" in reason for reason in decision.reasons) @@ -651,7 +634,7 @@ def test_target_prompt_paths_reject_same_resolved_file(tmp_path: Path): equivalent_path = tmp_path / "nested" / ".." / prompt_path.name prompt_path.write_text("baseline", encoding="utf-8") - with pytest.raises(ValueError, match="same resolved file"): + with pytest.raises(ValueError) as exc_info: _parse_target_prompt_paths( [ f"system_prompt={prompt_path}", @@ -660,6 +643,8 @@ def test_target_prompt_paths_reject_same_resolved_file(tmp_path: Path): default_prompt_path=prompt_path, ) + assert str(exc_info.value) == "--target-prompt fields must not reference the same resolved file" + @pytest.mark.parametrize( ("field_name", "field_value"), @@ -879,40 +864,8 @@ async def optimize(**kwargs): ], ) - class FakeAgentEvaluator: - @staticmethod - def get_executer(eval_dataset_file_path_or_dir, **kwargs): - return FakeEvalExecuter(Path(eval_dataset_file_path_or_dir)) - - class FakeEvalExecuter: - def __init__(self, eval_path: Path): - self.eval_path = eval_path - self._result = None - - async def evaluate(self): - split = "train" if "train" in self.eval_path.name else "validation" - prompt_label = _current_prompt_label(calls) - calls.setdefault("agent_evaluator_runs", []).append((prompt_label, split)) - scores = _fake_eval_scores(self.eval_path, split=split, prompt_label=prompt_label) - self._result = types.SimpleNamespace( - results_by_eval_set_id={ - f"{split}_set": types.SimpleNamespace( - eval_results_by_eval_id={ - case_id: [_fake_case_result(case_id, score)] - for case_id, score in scores - } - ) - } - ) - if any(score < 1.0 for _, score in scores): - raise AssertionError("evaluation cases failed") - - def get_result(self): - return self._result - fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") fake_eval_module.AgentOptimizer = FakeAgentOptimizer - fake_eval_module.AgentEvaluator = FakeAgentEvaluator fake_eval_module.TargetPrompt = FakeTargetPrompt monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) @@ -926,65 +879,6 @@ async def call_agent(query: str) -> str: return calls -def _current_prompt_label(calls: dict) -> str: - target_prompt = calls.get("target_prompt") - paths = getattr(target_prompt, "paths", []) if target_prompt is not None else [] - contents = [ - Path(path).read_text(encoding="utf-8") - for _, path in paths - if Path(path).is_file() - ] - return "sdk_best" if any("optimized" in content for content in contents) else "baseline" - - -def _fake_eval_scores(eval_path: Path, *, split: str, prompt_label: str) -> list[tuple[str, float]]: - case_ids = _case_ids(eval_path) - if not case_ids: - return [] - if prompt_label == "sdk_best": - return [(case_id, 1.0) for case_id in case_ids] - baseline_scores = { - "train_json_refund": 0.0, - "train_exact_order_status": 0.0, - "train_rubric_retry_summary": 1.0, - "val_json_invoice": 0.0, - "val_explain_cache": 1.0, - "val_protected_yes_no": 1.0, - } - return [(case_id, baseline_scores.get(case_id, 0.5 if split == "validation" else 0.0)) for case_id in case_ids] - - -def _case_ids(eval_path: Path) -> list[str]: - payload = json.loads(eval_path.read_text(encoding="utf-8")) - cases = payload.get("cases") or payload.get("eval_cases") or payload.get("evalCases") or [] - ids = [] - for case in cases: - if isinstance(case, dict): - case_id = case.get("id") or case.get("case_id") or case.get("eval_id") or case.get("evalId") - if case_id: - ids.append(str(case_id)) - return ids - - -def _fake_case_result(case_id: str, score: float): - passed = score >= 1.0 - status = "PASSED" if passed else "FAILED" - return types.SimpleNamespace( - eval_id=case_id, - final_eval_status=status, - error_message=None if passed else "response did not match expectation", - overall_eval_metric_results=[ - types.SimpleNamespace( - metric_name="response_match_score", - score=score, - eval_status=status, - details=types.SimpleNamespace(reason="response did not match expectation"), - ) - ], - eval_metric_result_per_invocation=[], - ) - - def _write_sdk_optimizer_config(tmp_path: Path) -> Path: path = tmp_path / "sdk_optimizer.json" path.write_text( From 7250d91db7681f6dbffabf28072355d2f03c3a74 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 19:38:04 +0800 Subject: [PATCH 14/34] fix(examples): honor disabled sdk cost gate --- .../eval_optimize_loop/run_pipeline.py | 11 +++++----- .../tests/test_sdk_backend.py | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 90863840..29f38ded 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -594,11 +594,12 @@ def _sdk_gate_decision( f"accept: validation improvement {improvement:.3f} meets threshold {min_improvement:.3f}" ) - if total_cost > max_cost: - accepted = False - reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}") - else: - reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}") + if max_cost is not None: + if total_cost > max_cost: + accepted = False + reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}") + else: + reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}") if accepted: reasons.append("accept: SDK aggregate gate passed") diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index d89dd4ee..7ae25294 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -14,6 +14,7 @@ from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL from examples.optimization.eval_optimize_loop.run_pipeline import _parse_target_prompt_paths +from examples.optimization.eval_optimize_loop.run_pipeline import _sdk_gate_decision from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline @@ -551,6 +552,26 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Pa assert any("cost" in reason for reason in decision.reasons) +def test_sdk_gate_decision_skips_disabled_cost_gate_but_keeps_quality_threshold(): + decision = _sdk_gate_decision( + candidate_id="sdk_best", + sdk_summary={ + "status": "SUCCEEDED", + "pass_rate_improvement": 0.005, + "total_llm_cost": 1000.0, + }, + gate_config={ + "min_val_score_improvement": 0.01, + "max_total_cost": None, + }, + ) + + assert decision.accepted is False + assert any("validation improvement" in reason for reason in decision.reasons) + assert not any("cost" in reason for reason in decision.reasons) + assert decision.total_run_cost == 1000.0 + + @pytest.mark.parametrize( ("field_name", "field_value"), [ From 623b52133a1a4ca11d72e608f68031e1b3179556 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 19:50:50 +0800 Subject: [PATCH 15/34] feat(examples): add transactional prompt writeback --- .../eval_optimize_loop/eval_loop/writeback.py | 205 ++++++++++++++ .../tests/test_writeback.py | 257 ++++++++++++++++++ 2 files changed, 462 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/writeback.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_writeback.py diff --git a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py new file mode 100644 index 00000000..625b5bb9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import hashlib +import os +import sys +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path + +from .schemas import WritebackResult + + +class ConcurrentPromptUpdateError(RuntimeError): + """Raised when source prompts no longer match their captured snapshot.""" + + +@dataclass(frozen=True) +class PromptFileSnapshot: + """Byte-for-byte snapshot of one source prompt file.""" + + name: str + path: Path + content: bytes + sha256: str + + +@dataclass(frozen=True) +class PromptSnapshot: + """Ordered bundle of source prompt snapshots keyed by prompt name.""" + + files: dict[str, PromptFileSnapshot] + + def hashes(self) -> dict[str, str]: + """Return a fresh name-to-hash mapping for this snapshot.""" + + return {name: prompt_file.sha256 for name, prompt_file in self.files.items()} + + +def _hash_bytes(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def snapshot_prompt_files(paths: dict[str, str | Path]) -> PromptSnapshot: + """Read source prompt files and capture their exact bytes and hashes.""" + + files: dict[str, PromptFileSnapshot] = {} + for name, raw_path in paths.items(): + path = Path(raw_path) + content = path.read_bytes() + files[name] = PromptFileSnapshot( + name=name, + path=path, + content=content, + sha256=_hash_bytes(content), + ) + return PromptSnapshot(files=files) + + +def _current_hashes(snapshot: PromptSnapshot) -> dict[str, str]: + return {name: _hash_bytes(prompt_file.path.read_bytes()) for name, prompt_file in snapshot.files.items()} + + +def _atomic_replace_bytes(path: Path, content: bytes) -> None: + """Replace one file atomically after durably flushing a same-directory temp.""" + + fd, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + temp_file = os.fdopen(fd, "wb") + fd = -1 + with temp_file: + temp_file.write(content) + temp_file.flush() + os.fsync(temp_file.fileno()) + os.replace(temp_path, path) + finally: + active_exception = sys.exc_info()[0] is not None + cleanup_error: OSError | None = None + if fd >= 0: + try: + os.close(fd) + except OSError as error: + cleanup_error = error + try: + temp_path.unlink() + except FileNotFoundError: + pass + except OSError as error: + if cleanup_error is None: + cleanup_error = error + if cleanup_error is not None and not active_exception: + raise cleanup_error + + +def _restore_snapshot(snapshot: PromptSnapshot) -> list[str]: + failures: list[str] = [] + for name, prompt_file in snapshot.files.items(): + try: + _atomic_replace_bytes(prompt_file.path, prompt_file.content) + except OSError as error: + failures.append(f"{name}: {error}") + return failures + + +def _encode_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> dict[str, bytes]: + missing = [name for name in snapshot.files if name not in prompts] + if missing: + raise ValueError(f"missing prompts for snapshot files: {', '.join(missing)}") + return {name: prompts[name].encode("utf-8") for name in snapshot.files} + + +def _restoration_error(snapshot: PromptSnapshot, failures: list[str]) -> str | None: + details: list[str] = [] + if failures: + details.append(f"restore failures: {'; '.join(failures)}") + + try: + current_hashes = _current_hashes(snapshot) + except OSError as error: + details.append(f"restore verification failed: {error}") + else: + expected_hashes = snapshot.hashes() + mismatched = [name for name, expected_hash in expected_hashes.items() if current_hashes[name] != expected_hash] + if mismatched: + details.append(f"restored hashes differ for: {', '.join(mismatched)}") + + if not details: + return None + return "; ".join(details) + + +@contextmanager +def temporary_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> Iterator[None]: + """Temporarily install candidate prompts and always restore the snapshot.""" + + encoded_prompts = _encode_prompt_bundle(snapshot, prompts) + try: + for name, prompt_file in snapshot.files.items(): + _atomic_replace_bytes(prompt_file.path, encoded_prompts[name]) + yield + finally: + failures = _restore_snapshot(snapshot) + restoration_error = _restoration_error(snapshot, failures) + if restoration_error is not None: + raise RuntimeError(f"failed to restore prompt snapshot: {restoration_error}") + + +def commit_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> WritebackResult: + """Apply a complete prompt bundle with CAS and compensating rollback.""" + + before_hashes = _current_hashes(snapshot) + expected_hashes = snapshot.hashes() + if before_hashes != expected_hashes: + changed = [name for name, expected_hash in expected_hashes.items() if before_hashes[name] != expected_hash] + raise ConcurrentPromptUpdateError(f"source prompt files changed since snapshot: {', '.join(changed)}") + + encoded_prompts = _encode_prompt_bundle(snapshot, prompts) + try: + for name, prompt_file in snapshot.files.items(): + _atomic_replace_bytes(prompt_file.path, encoded_prompts[name]) + applied_hashes = _current_hashes(snapshot) + except OSError as error: + rollback_failures = _restore_snapshot(snapshot) + try: + after_hashes = _current_hashes(snapshot) + except OSError as hash_error: + after_hashes = {} + rollback_failures.append(f"hash verification: {hash_error}") + else: + rollback_failures.extend( + f"{name}: restored hash differs from snapshot" + for name, expected_hash in expected_hashes.items() + if after_hashes[name] != expected_hash + ) + + error_message = f"prompt commit failed: {error}" + if rollback_failures: + error_message += f"; rollback failures: {'; '.join(rollback_failures)}" + return WritebackResult( + status="rollback_failed" if rollback_failures else "rolled_back", + before_hashes=before_hashes, + after_hashes=after_hashes, + error=error_message, + ) + + return WritebackResult( + status="applied", + before_hashes=before_hashes, + after_hashes=applied_hashes, + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_writeback.py b/examples/optimization/eval_optimize_loop/tests/test_writeback.py new file mode 100644 index 00000000..404dc477 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_writeback.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop import writeback +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( + ConcurrentPromptUpdateError, +) +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( + commit_prompt_bundle, +) +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( + snapshot_prompt_files, +) +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( + temporary_prompt_bundle, +) + + +def _prompt_files(tmp_path: Path) -> tuple[dict[str, Path], dict[str, bytes]]: + paths = { + "system": tmp_path / "system.txt", + "user": tmp_path / "user.txt", + } + baseline = { + "system": b"system baseline\r\n\xff", + "user": "user baseline \u57fa\u7ebf\n".encode(), + } + for name, path in paths.items(): + path.write_bytes(baseline[name]) + return paths, baseline + + +def test_snapshot_prompt_files_preserves_bytes_and_returns_hash_copy(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + + snapshot = snapshot_prompt_files(paths) + + system = snapshot.files["system"] + assert system.name == "system" + assert system.path == paths["system"] + assert system.content == baseline["system"] + assert system.sha256 == hashlib.sha256(baseline["system"]).hexdigest() + + hashes = snapshot.hashes() + hashes["system"] = "mutated copy" + assert snapshot.hashes()["system"] == system.sha256 + + +def test_temporary_prompt_bundle_restores_exact_bytes_after_body_error(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + + with pytest.raises(RuntimeError, match="candidate failed"): + with temporary_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate \u7528\u6237"}, + ): + assert paths["system"].read_bytes() == b"candidate system" + assert paths["user"].read_bytes() == "candidate \u7528\u6237".encode() + raise RuntimeError("candidate failed") + + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_temporary_prompt_bundle_rejects_missing_prompt_before_writing(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + + with pytest.raises(ValueError, match="user"): + with temporary_prompt_bundle(snapshot, {"system": "candidate"}): + pytest.fail("incomplete prompt bundle must not be entered") + + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_commit_rejects_concurrent_update_before_any_write(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"updated by another process" + paths["user"].write_bytes(external_content) + + def unexpected_write(path: Path, content: bytes) -> None: + pytest.fail(f"unexpected write to {path} with {content!r}") + + monkeypatch.setattr(writeback, "_atomic_replace_bytes", unexpected_write) + + with pytest.raises(ConcurrentPromptUpdateError, match="changed"): + commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_commit_rejects_missing_prompt_before_writing(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + + def unexpected_write(path: Path, content: bytes) -> None: + pytest.fail(f"unexpected write to {path} with {content!r}") + + monkeypatch.setattr(writeback, "_atomic_replace_bytes", unexpected_write) + + with pytest.raises(ValueError, match="user"): + commit_prompt_bundle(snapshot, {"system": "candidate"}) + + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_commit_applies_complete_bundle_and_reports_hashes(tmp_path: Path): + paths, _ = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + prompts = {"system": "new system", "user": "new \u7528\u6237"} + + result = commit_prompt_bundle(snapshot, prompts) + + assert result.status == "applied" + assert result.before_hashes == snapshot.hashes() + assert result.after_hashes == {name: hashlib.sha256(prompts[name].encode()).hexdigest() for name in paths} + assert result.error is None + assert {name: path.read_bytes() for name, path in paths.items()} == {name: prompts[name].encode() for name in paths} + + +def test_commit_rolls_back_when_post_write_hashing_fails(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + original_read_bytes = Path.read_bytes + hash_read_failed = False + + def fail_once_for_candidate_system(path: Path): + nonlocal hash_read_failed + content = original_read_bytes(path) + if not hash_read_failed and path == paths["system"] and content == b"candidate system": + hash_read_failed = True + raise OSError("post-write hash read failed") + return content + + monkeypatch.setattr(Path, "read_bytes", fail_once_for_candidate_system) + + result = commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert hash_read_failed + assert result.status == "rolled_back" + assert result.before_hashes == snapshot.hashes() + assert result.after_hashes == snapshot.hashes() + assert result.error is not None + assert "post-write hash read failed" in result.error + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_commit_rolls_back_all_files_when_second_replace_fails(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + original_replace = os.replace + replace_calls = 0 + + def fail_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 2: + raise OSError("second replace failed") + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "replace", fail_second_replace) + + result = commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert result.status == "rolled_back" + assert result.before_hashes == snapshot.hashes() + assert result.after_hashes == snapshot.hashes() + assert result.error is not None + assert "second replace failed" in result.error + assert replace_calls == 4 + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_commit_reports_rollback_failed_when_restored_hashes_differ(tmp_path: Path, monkeypatch): + paths, _ = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + original_replace = os.replace + original_restore = writeback._restore_snapshot + replace_calls = 0 + + def fail_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 2: + raise OSError("candidate write failed") + return original_replace(source, destination) + + def restore_then_concurrently_update(snapshot_to_restore): + failures = original_restore(snapshot_to_restore) + paths["system"].write_bytes(b"concurrent update after restore") + return failures + + monkeypatch.setattr(writeback.os, "replace", fail_second_replace) + monkeypatch.setattr(writeback, "_restore_snapshot", restore_then_concurrently_update) + + result = commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert result.status == "rollback_failed" + assert result.after_hashes["system"] != snapshot.hashes()["system"] + assert result.error is not None + assert "system: restored hash differs from snapshot" in result.error + + +def test_commit_reports_rollback_failed_and_attempts_every_restore(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + original_replace = os.replace + replace_calls = 0 + + def fail_write_and_first_restore( + source: str | bytes | Path, + destination: str | bytes | Path, + ): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 2: + raise OSError("candidate write failed") + if replace_calls == 3: + raise OSError("system restore failed") + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "replace", fail_write_and_first_restore) + + result = commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert result.status == "rollback_failed" + assert result.before_hashes == snapshot.hashes() + assert result.after_hashes["system"] != snapshot.hashes()["system"] + assert result.after_hashes["user"] == snapshot.hashes()["user"] + assert result.error is not None + assert "candidate write failed" in result.error + assert "system: system restore failed" in result.error + assert replace_calls == 4 + assert paths["system"].read_bytes() == b"candidate system" + assert paths["user"].read_bytes() == baseline["user"] From 5b482eaf385ab0beb9460c829e3e48f064333c0d Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 19:57:28 +0800 Subject: [PATCH 16/34] test(examples): cover writeback temp cleanup --- .../tests/test_writeback.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/examples/optimization/eval_optimize_loop/tests/test_writeback.py b/examples/optimization/eval_optimize_loop/tests/test_writeback.py index 404dc477..30e38bf8 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_writeback.py +++ b/examples/optimization/eval_optimize_loop/tests/test_writeback.py @@ -158,6 +158,57 @@ def fail_once_for_candidate_system(path: Path): assert {name: path.read_bytes() for name, path in paths.items()} == baseline +def test_commit_removes_temp_file_when_replace_fails(tmp_path: Path, monkeypatch): + target = tmp_path / "system.txt" + baseline = b"system baseline" + target.write_bytes(baseline) + snapshot = snapshot_prompt_files({"system": target}) + original_replace = os.replace + replace_calls = 0 + + def fail_first_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 1: + raise OSError("candidate replace failed") + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "replace", fail_first_replace) + + result = commit_prompt_bundle(snapshot, {"system": "candidate system"}) + + assert result.status == "rolled_back" + assert target.read_bytes() == baseline + assert list(tmp_path.glob(f".{target.name}.*.tmp")) == [] + + +def test_atomic_replace_preserves_replace_error_when_temp_unlink_fails(tmp_path: Path, monkeypatch): + target = tmp_path / "system.txt" + baseline = b"system baseline" + target.write_bytes(baseline) + original_unlink = Path.unlink + + def fail_replace(source: str | bytes | Path, destination: str | bytes | Path): + raise OSError("primary replace failed") + + def fail_temp_unlink(path: Path, missing_ok: bool = False): + if path.parent == tmp_path and path.name.startswith(f".{target.name}.") and path.suffix == ".tmp": + raise OSError("temp unlink failed") + return original_unlink(path, missing_ok=missing_ok) + + with monkeypatch.context() as injected_failure: + injected_failure.setattr(writeback.os, "replace", fail_replace) + injected_failure.setattr(Path, "unlink", fail_temp_unlink) + with pytest.raises(OSError) as error_info: + writeback._atomic_replace_bytes(target, b"candidate system") + + for temp_path in tmp_path.glob(f".{target.name}.*.tmp"): + original_unlink(temp_path) + + assert str(error_info.value) == "primary replace failed" + assert target.read_bytes() == baseline + + def test_commit_rolls_back_all_files_when_second_replace_fails(tmp_path: Path, monkeypatch): paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) From 2630b54591f714d6d66d4a17e36913e530f0e595 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 20:17:39 +0800 Subject: [PATCH 17/34] fix(examples): preserve concurrent prompt updates --- .../eval_optimize_loop/eval_loop/writeback.py | 106 ++++++-- .../tests/test_writeback.py | 231 +++++++++++++++++- 2 files changed, 313 insertions(+), 24 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py index 625b5bb9..e6b962ad 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py @@ -1,3 +1,9 @@ +"""Best-effort prompt CAS with per-file atomic replacement. + +Hash checks narrow concurrency windows but cannot make a multi-file operation +transactional. A non-cooperating writer can still race after the last check. +""" + from __future__ import annotations import hashlib @@ -62,8 +68,13 @@ def _current_hashes(snapshot: PromptSnapshot) -> dict[str, str]: return {name: _hash_bytes(prompt_file.path.read_bytes()) for name, prompt_file in snapshot.files.items()} -def _atomic_replace_bytes(path: Path, content: bytes) -> None: - """Replace one file atomically after durably flushing a same-directory temp.""" +def _atomic_replace_bytes( + path: Path, + content: bytes, + *, + expected_sha256: str | None = None, +) -> None: + """Replace one file atomically if its last observed hash is still expected.""" fd, temp_name = tempfile.mkstemp( dir=path.parent, @@ -78,6 +89,10 @@ def _atomic_replace_bytes(path: Path, content: bytes) -> None: temp_file.write(content) temp_file.flush() os.fsync(temp_file.fileno()) + if expected_sha256 is not None: + current_sha256 = _hash_bytes(path.read_bytes()) + if current_sha256 != expected_sha256: + raise ConcurrentPromptUpdateError(f"source prompt file changed before replace: {path}") os.replace(temp_path, path) finally: active_exception = sys.exc_info()[0] is not None @@ -98,13 +113,31 @@ def _atomic_replace_bytes(path: Path, content: bytes) -> None: raise cleanup_error -def _restore_snapshot(snapshot: PromptSnapshot) -> list[str]: +def _restore_snapshot( + snapshot: PromptSnapshot, + written_hashes: dict[str, str], +) -> list[str]: + """Conditionally restore only files this operation successfully replaced.""" + failures: list[str] = [] - for name, prompt_file in snapshot.files.items(): + for name, candidate_hash in written_hashes.items(): + prompt_file = snapshot.files[name] try: - _atomic_replace_bytes(prompt_file.path, prompt_file.content) + current_hash = _hash_bytes(prompt_file.path.read_bytes()) except OSError as error: - failures.append(f"{name}: {error}") + failures.append(f"{name}: restore precondition failed: {error}") + continue + if current_hash != candidate_hash: + failures.append(f"{name}: restore conflict; current hash changed from candidate") + continue + try: + _atomic_replace_bytes( + prompt_file.path, + prompt_file.content, + expected_sha256=candidate_hash, + ) + except (OSError, ConcurrentPromptUpdateError) as error: + failures.append(f"{name}: restore failed: {error}") return failures @@ -143,15 +176,39 @@ def temporary_prompt_bundle( snapshot: PromptSnapshot, prompts: dict[str, str], ) -> Iterator[None]: - """Temporarily install candidate prompts and always restore the snapshot.""" + """Install candidates temporarily using best-effort CAS and conditional restore.""" encoded_prompts = _encode_prompt_bundle(snapshot, prompts) + expected_hashes = snapshot.hashes() + current_hashes = _current_hashes(snapshot) + if current_hashes != expected_hashes: + changed = [name for name, expected_hash in expected_hashes.items() if current_hashes[name] != expected_hash] + raise ConcurrentPromptUpdateError(f"source prompt files changed since snapshot: {', '.join(changed)}") + + candidate_hashes = {name: _hash_bytes(content) for name, content in encoded_prompts.items()} + written_hashes: dict[str, str] = {} try: for name, prompt_file in snapshot.files.items(): - _atomic_replace_bytes(prompt_file.path, encoded_prompts[name]) + _atomic_replace_bytes( + prompt_file.path, + encoded_prompts[name], + expected_sha256=expected_hashes[name], + ) + written_hashes[name] = candidate_hashes[name] yield - finally: - failures = _restore_snapshot(snapshot) + except BaseException as primary_error: + failures = _restore_snapshot(snapshot, written_hashes) + restoration_error = _restoration_error(snapshot, failures) + if restoration_error is not None: + diagnostic = f"failed to restore prompt snapshot: {restoration_error}" + add_note = getattr(primary_error, "add_note", None) + if add_note is not None: + add_note(diagnostic) + else: + raise primary_error from RuntimeError(diagnostic) + raise + else: + failures = _restore_snapshot(snapshot, written_hashes) restoration_error = _restoration_error(snapshot, failures) if restoration_error is not None: raise RuntimeError(f"failed to restore prompt snapshot: {restoration_error}") @@ -161,7 +218,7 @@ def commit_prompt_bundle( snapshot: PromptSnapshot, prompts: dict[str, str], ) -> WritebackResult: - """Apply a complete prompt bundle with CAS and compensating rollback.""" + """Apply a bundle with best-effort CAS and conditional compensating rollback.""" before_hashes = _current_hashes(snapshot) expected_hashes = snapshot.hashes() @@ -170,12 +227,29 @@ def commit_prompt_bundle( raise ConcurrentPromptUpdateError(f"source prompt files changed since snapshot: {', '.join(changed)}") encoded_prompts = _encode_prompt_bundle(snapshot, prompts) + candidate_hashes = {name: _hash_bytes(content) for name, content in encoded_prompts.items()} + written_hashes: dict[str, str] = {} try: for name, prompt_file in snapshot.files.items(): - _atomic_replace_bytes(prompt_file.path, encoded_prompts[name]) + _atomic_replace_bytes( + prompt_file.path, + encoded_prompts[name], + expected_sha256=expected_hashes[name], + ) + written_hashes[name] = candidate_hashes[name] applied_hashes = _current_hashes(snapshot) - except OSError as error: - rollback_failures = _restore_snapshot(snapshot) + candidate_mismatches = [ + name for name, candidate_hash in candidate_hashes.items() if applied_hashes[name] != candidate_hash + ] + if candidate_mismatches: + raise ConcurrentPromptUpdateError( + "candidate prompt files changed before final verification: " f"{', '.join(candidate_mismatches)}" + ) + except (OSError, ConcurrentPromptUpdateError) as error: + if isinstance(error, ConcurrentPromptUpdateError) and not written_hashes: + raise + + rollback_failures = _restore_snapshot(snapshot, written_hashes) try: after_hashes = _current_hashes(snapshot) except OSError as hash_error: @@ -183,7 +257,7 @@ def commit_prompt_bundle( rollback_failures.append(f"hash verification: {hash_error}") else: rollback_failures.extend( - f"{name}: restored hash differs from snapshot" + f"{name}: final hash differs from snapshot" for name, expected_hash in expected_hashes.items() if after_hashes[name] != expected_hash ) @@ -192,7 +266,7 @@ def commit_prompt_bundle( if rollback_failures: error_message += f"; rollback failures: {'; '.join(rollback_failures)}" return WritebackResult( - status="rollback_failed" if rollback_failures else "rolled_back", + status="rolled_back" if after_hashes == expected_hashes else "rollback_failed", before_hashes=before_hashes, after_hashes=after_hashes, error=error_message, diff --git a/examples/optimization/eval_optimize_loop/tests/test_writeback.py b/examples/optimization/eval_optimize_loop/tests/test_writeback.py index 30e38bf8..8d6ae1cf 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_writeback.py +++ b/examples/optimization/eval_optimize_loop/tests/test_writeback.py @@ -78,6 +78,101 @@ def test_temporary_prompt_bundle_rejects_missing_prompt_before_writing(tmp_path: assert {name: path.read_bytes() for name, path in paths.items()} == baseline +def test_temporary_rejects_stale_snapshot_without_overwriting_external_update(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external before temporary entry" + paths["user"].write_bytes(external_content) + original_replace = os.replace + replace_calls = 0 + + def count_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "replace", count_replace) + + with pytest.raises(ConcurrentPromptUpdateError, match="changed"): + with temporary_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ): + pass + + assert replace_calls == 0 + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_temporary_restore_preserves_external_update_and_reports_conflict(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external during temporary evaluation" + + with pytest.raises(RuntimeError, match="restore"): + with temporary_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ): + paths["user"].write_bytes(external_content) + + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_temporary_body_error_remains_primary_when_restore_conflicts(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external before failed body exits" + + with pytest.raises(RuntimeError, match="candidate failed") as error_info: + with temporary_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ): + paths["user"].write_bytes(external_content) + raise RuntimeError("candidate failed") + + diagnostics = " ".join(getattr(error_info.value, "__notes__", [])) + if error_info.value.__cause__ is not None: + diagnostics += f" {error_info.value.__cause__}" + if error_info.value.__context__ is not None: + diagnostics += f" {error_info.value.__context__}" + + assert "restore" in diagnostics + assert "user" in diagnostics + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_temporary_partial_install_restores_written_file_and_preserves_external_update(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external before second temporary replace" + original_fsync = os.fsync + fsync_calls = 0 + + def inject_external_before_second_precondition(file_descriptor: int): + nonlocal fsync_calls + original_fsync(file_descriptor) + fsync_calls += 1 + if fsync_calls == 2: + paths["user"].write_bytes(external_content) + + monkeypatch.setattr(writeback.os, "fsync", inject_external_before_second_precondition) + + with pytest.raises(ConcurrentPromptUpdateError, match="changed"): + with temporary_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ): + pass + + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + def test_commit_rejects_concurrent_update_before_any_write(tmp_path: Path, monkeypatch): paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) @@ -99,6 +194,128 @@ def unexpected_write(path: Path, content: bytes) -> None: assert paths["user"].read_bytes() == external_content +def test_commit_rechecks_first_file_after_flush_before_replace(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external between bundle CAS and first replace" + original_fsync = os.fsync + external_injected = False + + def inject_external_before_precondition(file_descriptor: int): + nonlocal external_injected + original_fsync(file_descriptor) + if not external_injected: + paths["system"].write_bytes(external_content) + external_injected = True + + monkeypatch.setattr(writeback.os, "fsync", inject_external_before_precondition) + + with pytest.raises(ConcurrentPromptUpdateError, match="changed"): + commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert external_injected + assert paths["system"].read_bytes() == external_content + assert paths["user"].read_bytes() == baseline["user"] + + +def test_commit_rolls_back_written_file_and_preserves_external_unwritten_file(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external before second file replace" + original_fsync = os.fsync + fsync_calls = 0 + + def inject_external_before_second_precondition(file_descriptor: int): + nonlocal fsync_calls + original_fsync(file_descriptor) + fsync_calls += 1 + if fsync_calls == 2: + paths["user"].write_bytes(external_content) + + monkeypatch.setattr(writeback.os, "fsync", inject_external_before_second_precondition) + + result = commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert result.status == "rollback_failed" + assert result.error is not None + assert "changed" in result.error + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_commit_rejects_final_candidate_hash_mismatch_and_preserves_external_update(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external after final candidate replace" + original_replace = os.replace + replace_calls = 0 + + def update_after_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + original_replace(source, destination) + replace_calls += 1 + if replace_calls == 2: + paths["user"].write_bytes(external_content) + + monkeypatch.setattr(writeback.os, "replace", update_after_second_replace) + + result = commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert result.status == "rollback_failed" + assert result.after_hashes["user"] == hashlib.sha256(external_content).hexdigest() + assert result.error is not None + assert "candidate" in result.error + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_commit_restore_rechecks_candidate_hash_after_flush(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external between restore checks" + original_fsync = os.fsync + original_replace = os.replace + fsync_calls = 0 + replace_calls = 0 + + def inject_external_during_restore(file_descriptor: int): + nonlocal fsync_calls + original_fsync(file_descriptor) + fsync_calls += 1 + if fsync_calls == 3: + paths["system"].write_bytes(external_content) + + def fail_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 2: + raise OSError("second candidate replace failed") + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "fsync", inject_external_during_restore) + monkeypatch.setattr(writeback.os, "replace", fail_second_replace) + + result = commit_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ) + + assert result.status == "rollback_failed" + assert result.error is not None + assert "changed" in result.error + assert paths["system"].read_bytes() == external_content + assert paths["user"].read_bytes() == baseline["user"] + + def test_commit_rejects_missing_prompt_before_writing(tmp_path: Path, monkeypatch): paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) @@ -209,7 +426,7 @@ def fail_temp_unlink(path: Path, missing_ok: bool = False): assert target.read_bytes() == baseline -def test_commit_rolls_back_all_files_when_second_replace_fails(tmp_path: Path, monkeypatch): +def test_commit_rolls_back_written_file_when_second_replace_fails(tmp_path: Path, monkeypatch): paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) original_replace = os.replace @@ -234,7 +451,6 @@ def fail_second_replace(source: str | bytes | Path, destination: str | bytes | P assert result.after_hashes == snapshot.hashes() assert result.error is not None assert "second replace failed" in result.error - assert replace_calls == 4 assert {name: path.read_bytes() for name, path in paths.items()} == baseline @@ -252,8 +468,8 @@ def fail_second_replace(source: str | bytes | Path, destination: str | bytes | P raise OSError("candidate write failed") return original_replace(source, destination) - def restore_then_concurrently_update(snapshot_to_restore): - failures = original_restore(snapshot_to_restore) + def restore_then_concurrently_update(snapshot_to_restore, written_hashes): + failures = original_restore(snapshot_to_restore, written_hashes) paths["system"].write_bytes(b"concurrent update after restore") return failures @@ -268,10 +484,10 @@ def restore_then_concurrently_update(snapshot_to_restore): assert result.status == "rollback_failed" assert result.after_hashes["system"] != snapshot.hashes()["system"] assert result.error is not None - assert "system: restored hash differs from snapshot" in result.error + assert "system: final hash differs from snapshot" in result.error -def test_commit_reports_rollback_failed_and_attempts_every_restore(tmp_path: Path, monkeypatch): +def test_commit_reports_rollback_failed_when_written_file_restore_fails(tmp_path: Path, monkeypatch): paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) original_replace = os.replace @@ -302,7 +518,6 @@ def fail_write_and_first_restore( assert result.after_hashes["user"] == snapshot.hashes()["user"] assert result.error is not None assert "candidate write failed" in result.error - assert "system: system restore failed" in result.error - assert replace_calls == 4 + assert "system restore failed" in result.error assert paths["system"].read_bytes() == b"candidate system" assert paths["user"].read_bytes() == baseline["user"] From 97756fd8f3d015230c630dd06f5ec6c3287693fa Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 20:23:49 +0800 Subject: [PATCH 18/34] fix(examples): verify temporary prompt bundle --- .../eval_optimize_loop/eval_loop/writeback.py | 8 +++++ .../tests/test_writeback.py | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py index e6b962ad..3a93c940 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py @@ -195,6 +195,14 @@ def temporary_prompt_bundle( expected_sha256=expected_hashes[name], ) written_hashes[name] = candidate_hashes[name] + installed_hashes = _current_hashes(snapshot) + candidate_mismatches = [ + name for name, candidate_hash in candidate_hashes.items() if installed_hashes[name] != candidate_hash + ] + if candidate_mismatches: + raise ConcurrentPromptUpdateError( + "candidate prompt files changed before temporary evaluation: " f"{', '.join(candidate_mismatches)}" + ) yield except BaseException as primary_error: failures = _restore_snapshot(snapshot, written_hashes) diff --git a/examples/optimization/eval_optimize_loop/tests/test_writeback.py b/examples/optimization/eval_optimize_loop/tests/test_writeback.py index 8d6ae1cf..6e83a097 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_writeback.py +++ b/examples/optimization/eval_optimize_loop/tests/test_writeback.py @@ -173,6 +173,35 @@ def inject_external_before_second_precondition(file_descriptor: int): assert paths["user"].read_bytes() == external_content +def test_temporary_verifies_complete_candidate_bundle_before_entering_body(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external after second candidate replace" + original_replace = os.replace + replace_calls = 0 + body_entered = False + + def update_first_file_after_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + original_replace(source, destination) + replace_calls += 1 + if replace_calls == 2: + paths["system"].write_bytes(external_content) + + monkeypatch.setattr(writeback.os, "replace", update_first_file_after_second_replace) + + with pytest.raises((ConcurrentPromptUpdateError, RuntimeError)): + with temporary_prompt_bundle( + snapshot, + {"system": "candidate system", "user": "candidate user"}, + ): + body_entered = True + + assert not body_entered + assert paths["system"].read_bytes() == external_content + assert paths["user"].read_bytes() == baseline["user"] + + def test_commit_rejects_concurrent_update_before_any_write(tmp_path: Path, monkeypatch): paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) From 717a4133854072e8e1eae4e4d083b35575e1782f Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 20:47:34 +0800 Subject: [PATCH 19/34] refactor(examples): normalize optimization backends --- .../eval_optimize_loop/eval_loop/backends.py | 874 ++++++++++++++++-- .../eval_optimize_loop/eval_loop/evaluator.py | 2 + .../tests/test_sdk_backend.py | 576 +++++++++++- 3 files changed, 1337 insertions(+), 115 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 38709137..ddc5ebca 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -1,36 +1,141 @@ -"""Backend adapters for fake and SDK optimization paths.""" +"""Backend-neutral adapters for fake and SDK optimization paths.""" from __future__ import annotations import asyncio import importlib import math +from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path from typing import Any -from typing import Iterable +from typing import Protocol from .diffing import make_unified_diff from .evaluator import ExampleEvaluator from .fake_judge import FakeJudge from .fake_model import FakeModel +from .loader import load_eval_cases from .optimizer import FakeOptimizer from .schemas import CandidatePrompt +from .schemas import CaseResult +from .schemas import CostSummary from .schemas import EvalCase from .schemas import EvalResult +from .schemas import OptimizationResult +from .schemas import OptimizationRound +from .writeback import snapshot_prompt_files +from .writeback import temporary_prompt_bundle + + +class EvaluationBackend(Protocol): + """Common asynchronous evaluation contract.""" + + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + ... + + +class OptimizationBackend(Protocol): + """Common asynchronous optimization contract.""" + + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + ... @dataclass class FakeBackend: + """Deterministic backend used by the self-contained example.""" + seed: int = 91 trace_enabled: bool = False def __post_init__(self) -> None: - self._evaluator = ExampleEvaluator(FakeModel(seed=self.seed), FakeJudge(), trace_enabled=self.trace_enabled) self._optimizer = FakeOptimizer() - def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: - return self._evaluator.evaluate(prompt_id=prompt_id, prompt=prompt, cases=cases, split=split) + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + del artifact_dir + prompt = _required_system_prompt(prompts, context=f"cannot evaluate {prompt_id}") + cases = load_eval_cases(dataset_path, split=split) + evaluator = ExampleEvaluator( + FakeModel(seed=self.seed), + FakeJudge(), + trace_enabled=trace, + ) + return evaluator.evaluate( + prompt_id=prompt_id, + prompt=prompt, + cases=cases, + split=split, + ) + + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + del train_path, validation_path, config_path, artifact_dir + baseline_prompt = _required_system_prompt( + baseline_prompts, + context="cannot optimize fake prompt bundle", + ) + candidates = _normalize_fake_candidates(self._optimizer.propose(baseline_prompt)) + zero_cost = CostSummary(complete=True) + rounds = [ + OptimizationRound( + round_id=index, + candidate_id=candidate.candidate_id, + prompts=candidate.bundle(), + rationale=candidate.rationale, + metrics={}, + cost=zero_cost, + duration_seconds=0.0, + ) + for index, candidate in enumerate(candidates, start=1) + ] + return OptimizationResult( + candidates=candidates, + rounds=rounds, + cost=zero_cost, + raw_summary={ + "backend": "fake", + "baseline_prompt_id": baseline_train.prompt_id, + "failure_summary": _safe_jsonable(failure_summary), + }, + ) def optimize( self, @@ -41,24 +146,38 @@ def optimize( optimizer_config_path: str | Path, output_dir: str | Path, ) -> list[CandidatePrompt]: - return self._optimizer.propose(baseline_prompt) + """Compatibility wrapper for the pre-async fake pipeline.""" + + del train_path, val_path, optimizer_config_path, output_dir + return _normalize_fake_candidates(self._optimizer.propose(baseline_prompt)) -@dataclass class SDKBackend: - """Thin optimizer adapter around AgentOptimizer/TargetPrompt for SDK runs. + """Adapter around SDK AgentEvaluator, AgentOptimizer, and TargetPrompt. - SDK mode relies on AgentOptimizer's internal evaluation loop. It does not - implement the fake per-case ``evaluate`` API. + update_source is accepted only through legacy keyword forwarding so the + current synchronous pipeline can transition independently. It is never + stored or delegated; source writeback belongs to the wrapper after gating. """ - prompt_path: str | Path - call_agent_path: str | None = None - update_source: bool = False - target_prompt_paths: dict[str, str | Path] | None = None - last_result: Any | None = None - last_result_summary: dict[str, Any] | None = None - last_artifact_dir: str | None = None + def __init__( + self, + prompt_path: str | Path, + call_agent_path: str | None = None, + target_prompt_paths: dict[str, str | Path] | None = None, + **legacy_options: object, + ) -> None: + unexpected = sorted(set(legacy_options) - {"update_source"}) + if unexpected: + raise TypeError(f"unexpected SDKBackend options: {', '.join(unexpected)}") + self.prompt_path = prompt_path + self.call_agent_path = call_agent_path + self.target_prompt_paths = dict(target_prompt_paths) if target_prompt_paths else None + self.last_result: Any | None = None + self.last_result_summary: dict[str, Any] | None = None + self.last_artifact_dir: str | None = None + self.last_baseline_prompts: dict[str, str] | None = None + self.last_best_prompts: dict[str, str] | None = None def optimize( self, @@ -69,6 +188,8 @@ def optimize( optimizer_config_path: str | Path, output_dir: str | Path, ) -> list[CandidatePrompt]: + """Safely bridge old synchronous callers to the async implementation.""" + if _has_running_loop(): raise ValueError( "SDKBackend.optimize() cannot be called while an event loop is already running; " @@ -93,66 +214,251 @@ async def optimize_async( optimizer_config_path: str | Path, output_dir: str | Path, ) -> list[CandidatePrompt]: - if not self.call_agent_path: - raise ValueError( - "sdk mode requires --sdk-call-agent module:function. The callable must be async and compatible " - "with AgentOptimizer.optimize(call_agent=...). Also configure real model credentials required " - "by that callable, such as TRPC_AGENT_API_KEY/TRPC_AGENT_BASE_URL/TRPC_AGENT_MODEL_NAME." - ) - call_agent = _load_call_agent(self.call_agent_path) + """Compatibility async wrapper returning the historical candidate list.""" + + target_paths = self._target_prompt_paths() + if set(target_paths) == {"system_prompt"}: + baseline_prompts = {"system_prompt": baseline_prompt} + else: + baseline_prompts = _read_prompt_bundle(target_paths) + result = await self.optimize_candidates( + baseline_prompts=baseline_prompts, + baseline_train=EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ), + failure_summary={}, + train_path=train_path, + validation_path=val_path, + config_path=optimizer_config_path, + artifact_dir=output_dir, + ) + return result.candidates + + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + del baseline_train, failure_summary + call_agent = self._load_required_call_agent(for_evaluation=False) try: from trpc_agent_sdk.evaluation import AgentOptimizer from trpc_agent_sdk.evaluation import TargetPrompt except Exception as exc: # pragma: no cover - depends on optional SDK import health raise ValueError(f"sdk mode could not import AgentOptimizer/TargetPrompt: {exc}") from exc - target_prompt_paths = self._target_prompt_paths() + target_paths = self._target_prompt_paths() + baseline_bundle = _validated_prompt_bundle( + baseline_prompts, + target_paths, + context="baseline prompt bundle", + ) + source_bundle = _read_prompt_bundle(target_paths) + mismatched = sorted( + name for name in target_paths if baseline_bundle[name] != source_bundle[name] + ) + if mismatched: + raise ValueError( + "baseline prompt bundle does not match registered source prompt files: " + + ", ".join(mismatched) + ) + target_prompt = TargetPrompt() - for name, path in target_prompt_paths.items(): + for name, path in target_paths.items(): target_prompt.add_path(name, str(path)) - result = await AgentOptimizer.optimize( - config_path=str(optimizer_config_path), - call_agent=call_agent, - target_prompt=target_prompt, - train_dataset_path=str(train_path), - validation_dataset_path=str(val_path), - output_dir=str(output_dir), - update_source=self.update_source, - verbose=0, + + snapshot = snapshot_prompt_files(target_paths) + try: + sdk_result = await AgentOptimizer.optimize( + config_path=str(config_path), + call_agent=call_agent, + target_prompt=target_prompt, + train_dataset_path=str(train_path), + validation_dataset_path=str(validation_path), + output_dir=str(artifact_dir), + update_source=False, + verbose=0, + ) + finally: + changed_sources = _changed_snapshot_files(snapshot) + if changed_sources: + raise RuntimeError( + "AgentOptimizer modified source prompt files despite update_source=False: " + + ", ".join(changed_sources) + ) + + total_llm_cost = _finite_result_field( + "total_llm_cost", + getattr(sdk_result, "total_llm_cost", 0.0), ) - best_prompts = dict(getattr(result, "best_prompts", {}) or {}) - if not best_prompts: + best_raw = getattr(sdk_result, "best_prompts", None) + if not best_raw: raise ValueError("sdk mode completed but OptimizeResult.best_prompts was empty") - missing_fields = [name for name in target_prompt_paths if name not in best_prompts] - if missing_fields: - missing = ", ".join(sorted(missing_fields)) - raise ValueError( - "sdk mode completed but OptimizeResult.best_prompts is missing registered target fields: " - f"{missing}" + best_prompts = _validated_prompt_bundle( + best_raw, + target_paths, + context="OptimizeResult.best_prompts", + ) + + candidates: list[CandidatePrompt] = [] + rounds: list[OptimizationRound] = [] + seen_bundles: set[tuple[tuple[str, str], ...]] = set() + for index, sdk_round in enumerate(getattr(sdk_result, "rounds", []) or [], start=1): + round_id = _round_id(sdk_round, fallback=index) + candidate_id = f"sdk_round_{round_id:03d}" + round_raw_prompts = getattr(sdk_round, "candidate_prompts", {}) or {} + round_prompts = ( + _validated_prompt_bundle( + round_raw_prompts, + target_paths, + context=f"SDK round {round_id} candidate_prompts", + ) + if round_raw_prompts + else {} ) - empty_fields = [ - name - for name in target_prompt_paths - if not isinstance(best_prompts[name], str) or not best_prompts[name].strip() - ] - if empty_fields: - empty = ", ".join(sorted(empty_fields)) - raise ValueError( - "sdk mode completed but OptimizeResult.best_prompts contained empty registered target fields: " - f"{empty}" + rationale = str(getattr(sdk_round, "acceptance_reason", "") or "") + round_metrics = _finite_metric_map( + getattr(sdk_round, "metric_breakdown", {}) or {}, + context=f"SDK round {round_id} metric_breakdown", ) - self.last_result = result - self.last_result_summary = _summarize_sdk_result(result) - self.last_artifact_dir = str(output_dir) - baseline_prompts = _read_prompt_bundle(target_prompt_paths) - return [ - CandidatePrompt( - candidate_id="sdk_best", - prompt=_render_prompt_bundle(best_prompts), - rationale="Best prompt returned by AgentOptimizer.optimize.", - prompt_diff=_render_prompt_bundle_diff(baseline_prompts, best_prompts), + round_cost_value = _finite_number( + getattr(sdk_round, "round_llm_cost", 0.0), + context=f"SDK round {round_id} round_llm_cost", ) - ] + duration_seconds = _finite_number( + getattr(sdk_round, "duration_seconds", 0.0), + context=f"SDK round {round_id} duration_seconds", + ) + rounds.append( + OptimizationRound( + round_id=round_id, + candidate_id=candidate_id, + prompts=round_prompts, + rationale=rationale, + metrics=round_metrics, + cost=CostSummary( + optimizer=round_cost_value, + total=round_cost_value, + complete=False, + ), + duration_seconds=duration_seconds, + ) + ) + if round_prompts: + bundle_key = _prompt_bundle_key(round_prompts) + if bundle_key not in seen_bundles: + seen_bundles.add(bundle_key) + candidates.append( + _candidate_from_bundle( + candidate_id=candidate_id, + prompts=round_prompts, + rationale=rationale, + baseline_prompts=baseline_bundle, + ) + ) + + best_key = _prompt_bundle_key(best_prompts) + if best_key not in seen_bundles: + candidates.append( + _candidate_from_bundle( + candidate_id="sdk_best", + prompts=best_prompts, + rationale="Best prompt returned by AgentOptimizer.optimize.", + baseline_prompts=baseline_bundle, + ) + ) + + raw_summary = _summarize_sdk_result(sdk_result) + cost = CostSummary( + optimizer=total_llm_cost, + total=total_llm_cost, + complete=False, + ) + result = OptimizationResult( + candidates=candidates, + rounds=rounds, + cost=cost, + raw_summary=raw_summary, + ) + self.last_result = sdk_result + self.last_result_summary = raw_summary + self.last_artifact_dir = str(artifact_dir) + self.last_baseline_prompts = baseline_bundle + self.last_best_prompts = best_prompts + return result + + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + del trace + call_agent = self._load_required_call_agent(for_evaluation=True) + try: + from trpc_agent_sdk.evaluation import AgentEvaluator + except Exception as exc: # pragma: no cover - depends on optional SDK import health + raise ValueError(f"sdk mode could not import AgentEvaluator: {exc}") from exc + + target_paths = self._target_prompt_paths() + candidate_prompts = _validated_prompt_bundle( + prompts, + target_paths, + context=f"cannot evaluate {prompt_id}", + ) + expected_cases = load_eval_cases(dataset_path, split=split) + snapshot = snapshot_prompt_files(target_paths) + result: Any | None = None + with temporary_prompt_bundle(snapshot, candidate_prompts): + executer = AgentEvaluator.get_executer( + str(dataset_path), + call_agent=call_agent, + print_detailed_results=False, + print_summary_report=False, + eval_result_output_dir=str(artifact_dir), + ) + try: + await executer.evaluate() + except Exception: + result = executer.get_result() + if result is None: + raise + else: + result = executer.get_result() + + if result is None: + raise ValueError(f"AgentEvaluator returned no result for {dataset_path}") + return _eval_result_from_sdk_result( + result, + prompt_id=prompt_id, + split=split, + expected_cases=expected_cases, + ) + + def _load_required_call_agent(self, *, for_evaluation: bool): + if not self.call_agent_path: + suffix = " for AgentEvaluator runs" if for_evaluation else ( + ". The callable must be async and compatible with " + "AgentOptimizer.optimize(call_agent=...). Also configure real model credentials required " + "by that callable, such as TRPC_AGENT_API_KEY/TRPC_AGENT_BASE_URL/TRPC_AGENT_MODEL_NAME." + ) + raise ValueError(f"sdk mode requires --sdk-call-agent module:function{suffix}") + return _load_call_agent(self.call_agent_path) def _target_prompt_paths(self) -> dict[str, str | Path]: if self.target_prompt_paths: @@ -160,6 +466,31 @@ def _target_prompt_paths(self) -> dict[str, str | Path]: return {"system_prompt": self.prompt_path} +def _required_system_prompt(prompts: dict[str, str], *, context: str) -> str: + if "system_prompt" not in prompts: + raise ValueError(f"{context}: missing required prompt field 'system_prompt'") + prompt = prompts["system_prompt"] + if not isinstance(prompt, str) or not prompt.strip(): + raise ValueError(f"{context}: prompt field 'system_prompt' must be a non-empty string") + return prompt + + +def _normalize_fake_candidates(candidates: Iterable[CandidatePrompt]) -> list[CandidatePrompt]: + normalized: list[CandidatePrompt] = [] + for candidate in candidates: + bundle = candidate.bundle() + normalized.append( + CandidatePrompt( + candidate_id=candidate.candidate_id, + prompt=candidate.prompt, + rationale=candidate.rationale, + prompt_diff=candidate.prompt_diff, + prompt_fields=bundle, + ) + ) + return normalized + + def _load_call_agent(path: str): if ":" not in path: raise ValueError("--sdk-call-agent must use module:function format") @@ -184,47 +515,194 @@ def _has_running_loop() -> bool: return True +def _validated_prompt_bundle( + prompts: Any, + paths: dict[str, str | Path], + *, + context: str, +) -> dict[str, str]: + if not isinstance(prompts, dict): + raise ValueError(f"{context} must be a prompt mapping") + missing_fields = sorted(name for name in paths if name not in prompts) + if missing_fields: + if context == "OptimizeResult.best_prompts": + raise ValueError( + "sdk mode completed but OptimizeResult.best_prompts is missing registered target fields: " + + ", ".join(missing_fields) + ) + raise ValueError(f"{context} is missing registered target fields: {', '.join(missing_fields)}") + extra_fields = sorted(name for name in prompts if name not in paths) + if extra_fields: + raise ValueError(f"{context} contains unregistered target fields: {', '.join(extra_fields)}") + empty_fields = sorted( + name + for name in paths + if not isinstance(prompts[name], str) or not prompts[name].strip() + ) + if empty_fields: + if context == "OptimizeResult.best_prompts": + raise ValueError( + "sdk mode completed but OptimizeResult.best_prompts contained empty registered target fields: " + + ", ".join(empty_fields) + ) + raise ValueError(f"{context} contains empty registered target fields: {', '.join(empty_fields)}") + return {name: prompts[name] for name in paths} + + +def _read_prompt_bundle(paths: dict[str, str | Path]) -> dict[str, str]: + return { + name: Path(path).read_text(encoding="utf-8") + for name, path in paths.items() + } + + +def _changed_snapshot_files(snapshot: Any) -> list[str]: + changed: list[str] = [] + for name, prompt_file in snapshot.files.items(): + try: + current = prompt_file.path.read_bytes() + except OSError: + changed.append(name) + else: + if current != prompt_file.content: + changed.append(name) + return changed + + +def _round_id(round_record: Any, *, fallback: int) -> int: + value = getattr(round_record, "round", fallback) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError("SDK round id must be a positive integer") + return value + + +def _prompt_bundle_key(prompts: dict[str, str]) -> tuple[tuple[str, str], ...]: + return tuple(sorted(prompts.items())) + + +def _candidate_from_bundle( + *, + candidate_id: str, + prompts: dict[str, str], + rationale: str, + baseline_prompts: dict[str, str], +) -> CandidatePrompt: + return CandidatePrompt( + candidate_id=candidate_id, + prompt=_render_prompt_bundle(prompts), + rationale=rationale, + prompt_diff=_render_prompt_bundle_diff( + baseline_prompts, + prompts, + candidate_id=candidate_id, + ), + prompt_fields=dict(prompts), + ) + + def _summarize_sdk_result(result: Any) -> dict[str, Any]: return { + "schema_version": _safe_jsonable(getattr(result, "schema_version", None)), + "algorithm": _safe_jsonable(getattr(result, "algorithm", None)), "status": _safe_jsonable(getattr(result, "status", None)), - "baseline_pass_rate": _safe_result_field( - "baseline_pass_rate", getattr(result, "baseline_pass_rate", None) + "finish_reason": _safe_jsonable(getattr(result, "finish_reason", None)), + "stop_reason": _safe_jsonable(getattr(result, "stop_reason", None)), + "error_message": _safe_jsonable(getattr(result, "error_message", None)), + "baseline_pass_rate": _finite_result_field( + "baseline_pass_rate", + getattr(result, "baseline_pass_rate", 0.0), + ), + "best_pass_rate": _finite_result_field( + "best_pass_rate", + getattr(result, "best_pass_rate", 0.0), + ), + "pass_rate_improvement": _finite_result_field( + "pass_rate_improvement", + getattr(result, "pass_rate_improvement", 0.0), + ), + "baseline_metric_breakdown": _finite_metric_map( + getattr(result, "baseline_metric_breakdown", {}) or {}, + context="SDK OptimizeResult baseline_metric_breakdown", ), - "best_pass_rate": _safe_result_field("best_pass_rate", getattr(result, "best_pass_rate", None)), - "pass_rate_improvement": _safe_result_field( - "pass_rate_improvement", getattr(result, "pass_rate_improvement", None) + "best_metric_breakdown": _finite_metric_map( + getattr(result, "best_metric_breakdown", {}) or {}, + context="SDK OptimizeResult best_metric_breakdown", + ), + "metric_thresholds": _finite_metric_map( + getattr(result, "metric_thresholds", {}) or {}, + context="SDK OptimizeResult metric_thresholds", + ), + "per_metric_best_candidates": _safe_jsonable( + getattr(result, "per_metric_best_candidates", {}) + ), + "total_llm_cost": _finite_result_field( + "total_llm_cost", + getattr(result, "total_llm_cost", 0.0), ), - "baseline_metric_breakdown": _safe_jsonable(getattr(result, "baseline_metric_breakdown", {})), - "best_metric_breakdown": _safe_jsonable(getattr(result, "best_metric_breakdown", {})), - "metric_thresholds": _safe_jsonable(getattr(result, "metric_thresholds", {})), - "total_llm_cost": _safe_result_field("total_llm_cost", getattr(result, "total_llm_cost", 0.0)), "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})), - "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)), + "duration_seconds": _finite_result_field( + "duration_seconds", + getattr(result, "duration_seconds", 0.0), + ), "started_at": _safe_jsonable(getattr(result, "started_at", None)), + "finished_at": _safe_jsonable(getattr(result, "finished_at", None)), "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)), "baseline_prompts": _safe_jsonable(getattr(result, "baseline_prompts", {})), "best_prompts": _safe_jsonable(getattr(result, "best_prompts", {})), "rounds": [ - { - "validation_pass_rate": _safe_jsonable(getattr(round_record, "validation_pass_rate", None)), - "accepted": _safe_jsonable(getattr(round_record, "accepted", None)), - "failed_case_ids": _safe_jsonable(getattr(round_record, "failed_case_ids", [])), - "round_llm_cost": _safe_jsonable(getattr(round_record, "round_llm_cost", 0.0)), - "budget_used": _safe_jsonable(getattr(round_record, "budget_used", None)), - "budget_total": _safe_jsonable(getattr(round_record, "budget_total", None)), - } + _round_raw_summary(round_record) for round_record in getattr(result, "rounds", []) or [] ], + "extras": _safe_jsonable(getattr(result, "extras", {})), } -def _safe_result_field(field_name: str, value: Any) -> Any: +def _round_raw_summary(round_record: Any) -> dict[str, Any]: + serialized = _safe_jsonable(round_record) + if isinstance(serialized, dict): + return serialized + return { + "round": _safe_jsonable(getattr(round_record, "round", None)), + "candidate_prompts": _safe_jsonable(getattr(round_record, "candidate_prompts", {})), + "validation_pass_rate": _safe_jsonable( + getattr(round_record, "validation_pass_rate", None) + ), + "metric_breakdown": _safe_jsonable(getattr(round_record, "metric_breakdown", {})), + "accepted": _safe_jsonable(getattr(round_record, "accepted", None)), + "acceptance_reason": _safe_jsonable( + getattr(round_record, "acceptance_reason", "") + ), + "failed_case_ids": _safe_jsonable(getattr(round_record, "failed_case_ids", [])), + "round_llm_cost": _safe_jsonable(getattr(round_record, "round_llm_cost", 0.0)), + "duration_seconds": _safe_jsonable(getattr(round_record, "duration_seconds", 0.0)), + } + + +def _finite_result_field(field_name: str, value: Any) -> float: try: - return _safe_jsonable(value) + return _finite_number(value, context=f"SDK OptimizeResult field {field_name}") except ValueError as exc: raise ValueError(f"SDK OptimizeResult field {field_name} must be a finite number") from exc +def _finite_metric_map(value: Any, *, context: str) -> dict[str, float]: + if not isinstance(value, dict): + raise ValueError(f"{context} must be a metric mapping") + return { + str(name): _finite_number(score, context=f"{context}.{name}") + for name, score in value.items() + } + + +def _finite_number(value: Any, *, context: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{context} must be a finite number") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{context} must be a finite number") + return number + + def _safe_jsonable(value: Any) -> Any: if hasattr(value, "model_dump"): return _safe_jsonable(value.model_dump(mode="json")) @@ -236,20 +714,221 @@ def _safe_jsonable(value: Any) -> Any: return [_safe_jsonable(item) for item in value] if isinstance(value, float): if not math.isfinite(value): - raise ValueError("value must be a finite number") + raise ValueError("SDK result values must be finite") return value if isinstance(value, (str, int, bool)) or value is None: return value return repr(value) -def _read_prompt_bundle(paths: dict[str, str | Path]) -> dict[str, str]: +def _eval_result_from_sdk_result( + result: Any, + *, + prompt_id: str, + split: str, + expected_cases: Iterable[EvalCase], +) -> EvalResult: + expected_by_id: dict[str, EvalCase] = {} + expected_order: list[str] = [] + for expected_case in expected_cases: + case_id = str(expected_case.case_id) + if case_id in expected_by_id: + raise ValueError(f"expected cases contain duplicate case id: {case_id}") + expected_by_id[case_id] = expected_case + expected_order.append(case_id) + + sdk_runs_by_id: dict[str, list[Any]] = {} + results_by_eval_set_id = getattr(result, "results_by_eval_set_id", {}) or {} + for set_result in results_by_eval_set_id.values(): + eval_results_by_eval_id = getattr(set_result, "eval_results_by_eval_id", {}) or {} + for raw_eval_id, runs in eval_results_by_eval_id.items(): + eval_id = str(raw_eval_id) + if eval_id in sdk_runs_by_id: + raise ValueError(f"SDK evaluation result contains duplicate case id: {eval_id}") + sdk_runs_by_id[eval_id] = list(runs or []) + + expected_ids = set(expected_by_id) + sdk_ids = set(sdk_runs_by_id) + if expected_ids != sdk_ids: + details: list[str] = [] + missing = sorted(expected_ids - sdk_ids) + extra = sorted(sdk_ids - expected_ids) + if missing: + details.append("missing SDK result IDs: " + ", ".join(missing)) + if extra: + details.append("extra SDK result IDs: " + ", ".join(extra)) + raise ValueError("SDK evaluation case IDs do not match expected cases; " + "; ".join(details)) + + case_results: list[CaseResult] = [] + for case_id in expected_order: + expected_case = expected_by_id[case_id] + run_list = sdk_runs_by_id[case_id] + metrics = _aggregate_case_metrics(run_list, case_id=case_id) + if metrics: + score = _mean(list(metrics.values())) + else: + score = _mean([ + 1.0 if _status_passed(getattr(run, "final_eval_status", None)) else 0.0 + for run in run_list + ]) + score = round(score, 6) + passed = bool(run_list) and all( + _status_passed(getattr(run, "final_eval_status", None)) + for run in run_list + ) + failure_reason, evidence, failure_category = _failure_details(run_list) + actual_invocation = _last_actual_invocation(run_list) + trace_available = actual_invocation is not None + trace_payload = ( + { + "user_content": _safe_jsonable( + getattr(actual_invocation, "user_content", None) + ), + "final_response": _safe_jsonable( + getattr(actual_invocation, "final_response", None) + ), + "intermediate_data": _safe_jsonable( + getattr(actual_invocation, "intermediate_data", None) + ), + } + if actual_invocation is not None + else {} + ) + output = ( + _content_text(getattr(actual_invocation, "final_response", None)) + if actual_invocation is not None + else "" + ) + case_results.append( + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=passed, + output=output, + metrics=metrics, + trace=trace_payload, + trace_available=trace_available, + failure_category=None if passed else failure_category, + failure_reason=None if passed else failure_reason, + evidence=None if passed else evidence, + cost=0.0, + hard_failed=(not passed and score <= 0.0), + expected_failure_category=expected_case.expected_failure_category, + ) + ) + + aggregate_score = ( + round(_mean([case.score for case in case_results]), 6) + if case_results + else 0.0 + ) + return EvalResult( + prompt_id=prompt_id, + split=split, + score=aggregate_score, + passed=all(case.passed for case in case_results), + cost=0.0, + cases=case_results, + ) + + +def _aggregate_case_metrics(runs: list[Any], *, case_id: str) -> dict[str, float]: + scores_by_metric: dict[str, list[float]] = {} + for run in runs: + for metric in getattr(run, "overall_eval_metric_results", []) or []: + raw_score = getattr(metric, "score", None) + if raw_score is None: + continue + metric_name = str(getattr(metric, "metric_name", "") or "") + if not metric_name: + raise ValueError(f"SDK case {case_id} contains a scored metric without a name") + score = _finite_number( + raw_score, + context=f"SDK case {case_id} metric {metric_name} score", + ) + scores_by_metric.setdefault(metric_name, []).append(score) return { - name: Path(path).read_text(encoding="utf-8") - for name, path in paths.items() + metric_name: round(_mean(scores), 6) + for metric_name, scores in scores_by_metric.items() } +def _mean(values: list[float]) -> float: + return sum(values) / len(values) if values else 0.0 + + +def _status_passed(status: Any) -> bool: + name = getattr(status, "name", None) + if name: + return str(name).upper() == "PASSED" + return str(status).split(".")[-1].upper() == "PASSED" + + +def _failure_details(runs: list[Any]) -> tuple[str, str, str]: + for run in runs: + if _status_passed(getattr(run, "final_eval_status", None)): + continue + error_message = getattr(run, "error_message", None) + for metric in getattr(run, "overall_eval_metric_results", []) or []: + if _status_passed(getattr(metric, "eval_status", None)): + continue + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details is not None else None + metric_name = str(getattr(metric, "metric_name", "") or "") + score = getattr(metric, "score", None) + evidence = f"{metric_name} score={score}" if metric_name else f"score={score}" + return ( + str(reason or error_message or "evaluation metric failed"), + evidence, + _metric_failure_category(metric_name), + ) + if error_message: + return (str(error_message), str(error_message), "evaluation_error") + return ("evaluation failed", "no failed metric detail available", "unknown_failure") + + +def _metric_failure_category(metric_name: str) -> str: + lowered = metric_name.lower() + if "parameter" in lowered or "arg" in lowered: + return "parameter_error" + if "tool" in lowered: + return "tool_call_error" + if "knowledge" in lowered or "recall" in lowered: + return "knowledge_recall_insufficient" + if "rubric" in lowered or "judge" in lowered or "llm" in lowered: + return "llm_rubric_not_met" + if "format" in lowered: + return "format_violation" + if "response" in lowered or "match" in lowered or "exact" in lowered: + return "final_response_mismatch" + return "unknown_failure" + + +def _last_actual_invocation(runs: list[Any]) -> Any | None: + for run in reversed(runs): + invocation_results = getattr(run, "eval_metric_result_per_invocation", []) or [] + for invocation_result in reversed(invocation_results): + actual = getattr(invocation_result, "actual_invocation", None) + if actual is not None: + return actual + return None + + +def _content_text(content: Any) -> str: + if content is None: + return "" + if hasattr(content, "model_dump"): + content = content.model_dump(mode="json") + if not isinstance(content, dict): + return "" + texts = [] + for part in content.get("parts") or []: + if isinstance(part, dict) and part.get("text") is not None: + texts.append(str(part["text"])) + return "\n".join(texts) + + def _render_prompt_bundle(prompts: dict[str, str]) -> str: if set(prompts) == {"system_prompt"}: return prompts["system_prompt"] @@ -259,22 +938,27 @@ def _render_prompt_bundle(prompts: dict[str, str]) -> str: return "\n\n".join(sections) -def _render_prompt_bundle_diff(baseline_prompts: dict[str, str], best_prompts: dict[str, str]) -> str: - if set(baseline_prompts) == {"system_prompt"} and set(best_prompts) == {"system_prompt"}: +def _render_prompt_bundle_diff( + baseline_prompts: dict[str, str], + candidate_prompts: dict[str, str], + *, + candidate_id: str, +) -> str: + if set(baseline_prompts) == {"system_prompt"} and set(candidate_prompts) == {"system_prompt"}: return make_unified_diff( baseline_prompts.get("system_prompt", ""), - best_prompts.get("system_prompt", ""), + candidate_prompts.get("system_prompt", ""), before_name="baseline_system_prompt.txt", - after_name="sdk_best/system_prompt.txt", + after_name=f"{candidate_id}/system_prompt.txt", ) diffs = [] - for name in sorted(set(baseline_prompts) | set(best_prompts)): + for name in sorted(set(baseline_prompts) | set(candidate_prompts)): diffs.append( make_unified_diff( baseline_prompts.get(name, ""), - best_prompts.get(name, ""), + candidate_prompts.get(name, ""), before_name=f"baseline/{name}.txt", - after_name=f"sdk_best/{name}.txt", + after_name=f"{candidate_id}/{name}.txt", ) ) return "\n\n".join(diffs) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py index e8a271ec..4bd0a615 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py @@ -53,7 +53,9 @@ def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], sp score=round(judged.score, 6), passed=judged.passed, output=output, + metrics={"fake_judge_score": round(judged.score, 6)}, trace=trace, + trace_available=bool(trace), failure_category=failure_category, failure_reason=failure_reason, evidence=evidence, diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 7ae25294..941c6759 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -1,6 +1,7 @@ from __future__ import annotations import builtins +import inspect import json import sys import types @@ -8,7 +9,11 @@ import pytest +from examples.optimization.eval_optimize_loop.eval_loop import backends as backend_module from examples.optimization.eval_optimize_loop.eval_loop.backends import SDKBackend +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationResult from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN @@ -18,6 +23,370 @@ from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline +def test_backend_protocols_expose_unified_async_api(): + assert inspect.iscoroutinefunction(backend_module.EvaluationBackend.evaluate) + assert inspect.iscoroutinefunction(backend_module.OptimizationBackend.optimize_candidates) + assert inspect.iscoroutinefunction(backend_module.FakeBackend.evaluate) + assert inspect.iscoroutinefunction(backend_module.FakeBackend.optimize_candidates) + assert inspect.iscoroutinefunction(backend_module.SDKBackend.evaluate) + assert inspect.iscoroutinefunction(backend_module.SDKBackend.optimize_candidates) + + +@pytest.mark.asyncio +async def test_fake_backend_implements_unified_contract_with_real_trace(tmp_path: Path): + prompt = "baseline prompt" + dataset_path = tmp_path / "fake_train.evalset.json" + dataset_path.write_text( + json.dumps({ + "split": "train", + "cases": [{ + "case_id": "case_a", + "input": "Mention latency and retries.", + "expectation": { + "type": "rubric", + "must_include": ["latency", "retries"], + }, + }], + }), + encoding="utf-8", + ) + backend = backend_module.FakeBackend(seed=91) + + result = await backend.evaluate( + prompt_id="baseline", + prompts={"system_prompt": prompt}, + dataset_path=dataset_path, + split="train", + trace=True, + artifact_dir=tmp_path / "fake_eval", + ) + + assert result.cases + assert all(case.metrics == {"fake_judge_score": case.score} for case in result.cases) + assert all(case.trace_available is True and case.trace for case in result.cases) + + without_trace = await backend.evaluate( + prompt_id="baseline_without_trace", + prompts={"system_prompt": prompt}, + dataset_path=dataset_path, + split="train", + trace=False, + artifact_dir=tmp_path / "fake_eval_without_trace", + ) + assert all(case.trace_available is False and not case.trace for case in without_trace.cases) + + with pytest.raises(ValueError, match="system_prompt"): + await backend.evaluate( + prompt_id="missing_system_prompt", + prompts={}, + dataset_path=dataset_path, + split="train", + trace=False, + artifact_dir=tmp_path / "missing", + ) + + +@pytest.mark.asyncio +async def test_fake_backend_wraps_candidates_in_complete_optimization_result(tmp_path: Path): + baseline_prompt = DEFAULT_PROMPT.read_text(encoding="utf-8") + baseline_train = _empty_eval_result("baseline", "train") + + result = await backend_module.FakeBackend(seed=91).optimize_candidates( + baseline_prompts={"system_prompt": baseline_prompt}, + baseline_train=baseline_train, + failure_summary={"failed_case_ids": ["train_json_strict"]}, + train_path=DEFAULT_TRAIN, + validation_path=DEFAULT_VAL, + config_path=DEFAULT_OPTIMIZER_CONFIG, + artifact_dir=tmp_path / "fake_optimize", + ) + + assert isinstance(result, OptimizationResult) + assert result.candidates + assert result.cost.complete is True + assert result.cost.total == 0.0 + assert len(result.rounds) == len(result.candidates) + assert all(candidate.bundle() == {"system_prompt": candidate.prompt} for candidate in result.candidates) + + +@pytest.mark.asyncio +async def test_sdk_backend_maps_rounds_deduplicates_bundles_and_marks_cost_incomplete( + tmp_path: Path, + monkeypatch, +): + round_prompts = { + "system_prompt": "round system", + "router_prompt": "round router", + } + rounds = [ + _sdk_round( + 1, + round_prompts, + acceptance_reason="first proposal", + metric_breakdown={"quality": 0.6}, + round_llm_cost=0.1, + duration_seconds=0.5, + accepted=False, + ), + _sdk_round( + 2, + dict(round_prompts), + acceptance_reason="duplicate proposal", + metric_breakdown={"quality": 0.7}, + round_llm_cost=0.2, + duration_seconds=0.75, + accepted=True, + ), + ] + calls = _install_fake_sdk( + monkeypatch, + best_prompts=round_prompts, + rounds=rounds, + total_llm_cost=0.3, + ) + system_path = tmp_path / "system.txt" + router_path = tmp_path / "router.txt" + system_path.write_text("baseline system", encoding="utf-8") + router_path.write_text("baseline router", encoding="utf-8") + baseline_prompts = { + "system_prompt": "baseline system", + "router_prompt": "baseline router", + } + backend = SDKBackend( + prompt_path=system_path, + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={"system_prompt": system_path, "router_prompt": router_path}, + ) + + result = await backend.optimize_candidates( + baseline_prompts=baseline_prompts, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={"failed_case_ids": ["case_a"]}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + assert calls["update_source"] is False + assert [candidate.candidate_id for candidate in result.candidates] == ["sdk_round_001"] + assert result.candidates[0].prompt_fields == round_prompts + assert result.candidates[0].bundle() == round_prompts + assert result.candidates[0].rationale == "first proposal" + assert "router_prompt" in result.candidates[0].prompt_diff + assert [round_record.round_id for round_record in result.rounds] == [1, 2] + assert result.rounds[0].metrics == {"quality": 0.6} + assert result.rounds[1].cost.total == 0.2 + assert result.cost.total == 0.3 + assert result.cost.complete is False + assert result.raw_summary["rounds"][1]["acceptance_reason"] == "duplicate proposal" + assert result.raw_summary["rounds"][1]["accepted"] is True + + +@pytest.mark.asyncio +async def test_sdk_backend_appends_best_when_no_round_contains_it(tmp_path: Path, monkeypatch): + round_prompts = {"system_prompt": "round prompt"} + best_prompts = {"system_prompt": "best prompt"} + _install_fake_sdk( + monkeypatch, + best_prompts=best_prompts, + rounds=[_sdk_round(4, round_prompts, acceptance_reason="explored")], + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + result = await SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + assert [candidate.candidate_id for candidate in result.candidates] == ["sdk_round_004", "sdk_best"] + assert result.candidates[1].prompt_fields == best_prompts + + +def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): + expected_case = EvalCase( + case_id="case_a", + split="validation", + input="question", + expectation={"answer": "expected"}, + expected_failure_category="format_violation", + ) + first_run = _sdk_case_run( + "case_a", + status="FAILED", + metrics=[ + _sdk_metric("response_match", 0.5, status="FAILED", reason="wrong response"), + _sdk_metric("style", 0.25, status="PASSED"), + ], + output="first output", + user_content="first question", + intermediate_data={"step": 1}, + ) + last_run = _sdk_case_run( + "case_a", + status="FAILED", + metrics=[ + _sdk_metric("response_match", 1.0, status="PASSED"), + _sdk_metric("style", 0.75, status="PASSED"), + ], + output="last output", + user_content="last question", + intermediate_data={"step": 2}, + ) + + mapped = backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result({"case_a": [first_run, last_run]}), + prompt_id="candidate", + split="validation", + expected_cases=[expected_case], + ) + + case = mapped.cases[0] + assert case.metrics == {"response_match": 0.75, "style": 0.5} + assert case.score == 0.625 + assert case.output == "last output" + assert case.trace_available is True + assert case.trace == { + "user_content": {"parts": [{"text": "last question"}]}, + "final_response": {"parts": [{"text": "last output"}]}, + "intermediate_data": {"step": 2}, + } + assert case.expected_failure_category == "format_violation" + assert case.failure_category == "final_response_mismatch" + assert case.failure_reason == "wrong response" + assert case.hard_failed is False + + +@pytest.mark.parametrize( + ("sdk_case_ids", "expected_case_ids", "message"), + [ + (["case_a"], ["case_a", "case_b"], "missing.*case_b"), + (["case_a", "case_b"], ["case_a"], "extra.*case_b"), + ], +) +def test_sdk_result_mapping_rejects_case_id_set_mismatch(sdk_case_ids, expected_case_ids, message): + sdk_runs = { + case_id: [_sdk_case_run(case_id, status="PASSED", metrics=[])] + for case_id in sdk_case_ids + } + expected_cases = [ + EvalCase(case_id=case_id, split="validation", input="", expectation={}) + for case_id in expected_case_ids + ] + + with pytest.raises(ValueError, match=message): + backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result(sdk_runs), + prompt_id="candidate", + split="validation", + expected_cases=expected_cases, + ) + + +def test_sdk_result_mapping_rejects_duplicate_case_ids_across_eval_sets(): + run = _sdk_case_run("case_a", status="PASSED", metrics=[]) + result = types.SimpleNamespace( + results_by_eval_set_id={ + "set_a": types.SimpleNamespace(eval_results_by_eval_id={"case_a": [run]}), + "set_b": types.SimpleNamespace(eval_results_by_eval_id={"case_a": [run]}), + } + ) + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + + with pytest.raises(ValueError, match="duplicate.*case_a"): + backend_module._eval_result_from_sdk_result( + result, + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_result_mapping_rejects_non_finite_metric_scores(): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + run = _sdk_case_run( + "case_a", + status="FAILED", + metrics=[_sdk_metric("quality", float("nan"), status="FAILED")], + ) + + with pytest.raises(ValueError, match="finite"): + backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result({"case_a": [run]}), + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +@pytest.mark.asyncio +async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_bytes( + tmp_path: Path, + monkeypatch, +): + dataset_path = tmp_path / "validation.evalset.json" + dataset_path.write_text( + json.dumps({ + "split": "validation", + "cases": [{ + "case_id": "case_a", + "input": "question", + "expectation": {"answer": "answer"}, + "expected_failure_category": "format_violation", + }], + }), + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + original_bytes = b"original prompt\r\n" + prompt_path.write_bytes(original_bytes) + sdk_result = _sdk_evaluate_result({ + "case_a": [ + _sdk_case_run( + "case_a", + status="PASSED", + metrics=[_sdk_metric("quality", 1.0, status="PASSED")], + output="answer", + user_content="question", + intermediate_data={"tool": "none"}, + ) + ] + }) + calls = _install_fake_agent_evaluator( + monkeypatch, + result=sdk_result, + on_evaluate=lambda: prompt_path.read_text(encoding="utf-8") == "candidate prompt", + raise_after_evaluate=True, + ) + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ) + + mapped = await backend.evaluate( + prompt_id="candidate", + prompts={"system_prompt": "candidate prompt"}, + dataset_path=dataset_path, + split="validation", + trace=False, + artifact_dir=tmp_path / "sdk_eval", + ) + + assert calls["observed_candidate"] is True + assert calls["eval_result_output_dir"] == str(tmp_path / "sdk_eval") + assert prompt_path.read_bytes() == original_bytes + assert mapped.cases[0].trace_available is True + + def test_sdk_backend_requires_call_agent_path(tmp_path: Path): backend = SDKBackend(prompt_path=tmp_path / "prompt.txt") @@ -157,24 +526,32 @@ def test_sdk_backend_empty_best_prompts_dict_error_is_clear(tmp_path: Path, monk ) -def test_sdk_backend_passes_update_source_true(tmp_path: Path, monkeypatch): - calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") +def test_sdk_backend_never_delegates_source_writeback(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + write_source_when_requested=True, + ) prompt_path = tmp_path / "prompt.txt" - prompt_path.write_text("baseline", encoding="utf-8") + original_bytes = b"baseline\r\n" + prompt_path.write_bytes(original_bytes) - SDKBackend( + backend = SDKBackend( prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent", update_source=True, - ).optimize( - baseline_prompt="baseline", + ) + backend.optimize( + baseline_prompt="baseline\n", train_path=tmp_path / "train.evalset.json", val_path=tmp_path / "val.evalset.json", optimizer_config_path=tmp_path / "optimizer.json", output_dir=tmp_path / "out", ) - assert calls["update_source"] is True + assert calls["update_source"] is False + assert prompt_path.read_bytes() == original_bytes + assert "update_source" not in vars(backend) def test_sdk_backend_call_agent_import_failure_names_target(tmp_path: Path): @@ -798,7 +1175,7 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: assert "--gate-config" in command -def test_run_pipeline_mode_sdk_passes_update_source_true(tmp_path: Path, monkeypatch): +def test_run_pipeline_mode_sdk_keeps_source_writeback_outside_backend(tmp_path: Path, monkeypatch): calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") report = run_pipeline( @@ -813,7 +1190,7 @@ def test_run_pipeline_mode_sdk_passes_update_source_true(tmp_path: Path, monkeyp ) assert report.run["update_source"] is True - assert calls["update_source"] is True + assert calls["update_source"] is False assert "--update-source" in report.run["reproducibility_command"] @@ -841,6 +1218,8 @@ def _install_fake_sdk( total_llm_cost: float = 0.123, duration_seconds: float = 12.3, started_at: str | None = None, + rounds: list[object] | None = None, + write_source_when_requested: bool = False, ): calls = {} @@ -856,9 +1235,24 @@ class FakeAgentOptimizer: @staticmethod async def optimize(**kwargs): calls.update(kwargs) + if write_source_when_requested and kwargs.get("update_source"): + for _, path in kwargs["target_prompt"].paths: + Path(path).write_bytes(b"optimizer mutated source") result_prompts = best_prompts if best_prompts is not None else { "system_prompt": "optimized prompt" if best_prompt is None else best_prompt } + effective_rounds = rounds if rounds is not None else [ + _sdk_round( + 1, + {}, + acceptance_reason="accepted", + validation_pass_rate=best_pass_rate, + accepted=True, + failed_case_ids=["case_a"], + round_llm_cost=total_llm_cost, + duration_seconds=duration_seconds, + ) + ] return types.SimpleNamespace( best_prompts=result_prompts, status=status, @@ -872,17 +1266,8 @@ async def optimize(**kwargs): total_token_usage={"prompt": 100, "completion": 25, "total": 125}, duration_seconds=duration_seconds, started_at=started_at, - total_rounds=1, - rounds=[ - types.SimpleNamespace( - validation_pass_rate=best_pass_rate, - accepted=True, - failed_case_ids=["case_a"], - round_llm_cost=total_llm_cost, - budget_used=3, - budget_total=10, - ) - ], + total_rounds=len(effective_rounds), + rounds=effective_rounds, ) fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") @@ -900,6 +1285,157 @@ async def call_agent(query: str) -> str: return calls +def _empty_eval_result(prompt_id: str, split: str) -> EvalResult: + return EvalResult( + prompt_id=prompt_id, + split=split, + score=0.0, + passed=False, + cost=0.0, + cases=[], + ) + + +def _sdk_round( + round_id: int, + candidate_prompts: dict[str, str], + *, + acceptance_reason: str = "", + metric_breakdown: dict[str, float] | None = None, + validation_pass_rate: float = 0.0, + round_llm_cost: float = 0.0, + duration_seconds: float = 0.0, + accepted: bool = False, + failed_case_ids: list[str] | None = None, +): + return types.SimpleNamespace( + round=round_id, + optimized_field_names=list(candidate_prompts), + candidate_prompts=dict(candidate_prompts), + train_pass_rate=0.0, + validation_pass_rate=validation_pass_rate, + metric_breakdown=dict(metric_breakdown or {}), + accepted=accepted, + acceptance_reason=acceptance_reason, + failed_case_ids=list(failed_case_ids or []), + failed_cases_truncated=0, + per_field_diagnosis={}, + reflection_lm_calls=1, + round_llm_cost=round_llm_cost, + round_token_usage={"prompt": 0, "completion": 0, "total": 0}, + started_at="2026-07-04T12:00:00+00:00", + duration_seconds=duration_seconds, + kind="reflective", + train_minibatch_size=0, + train_subsample_parent_score=None, + train_subsample_candidate_score=None, + skip_reason=None, + error_message=None, + budget_used=3, + budget_total=10, + extras={}, + ) + + +def _sdk_metric(metric_name: str, score: float, *, status: str, reason: str | None = None): + return types.SimpleNamespace( + metric_name=metric_name, + threshold=0.5, + score=score, + eval_status=status, + details=types.SimpleNamespace(reason=reason, score=score, rubric_scores=None), + ) + + +def _sdk_case_run( + case_id: str, + *, + status: str, + metrics: list[object], + output: str | None = None, + user_content: str | None = None, + intermediate_data: object | None = None, +): + per_invocation = [] + if output is not None or user_content is not None or intermediate_data is not None: + actual_invocation = types.SimpleNamespace( + invocation_id=f"{case_id}_invocation", + user_content={"parts": [{"text": user_content or ""}]}, + final_response={"parts": [{"text": output or ""}]} if output is not None else None, + intermediate_data=intermediate_data, + creation_timestamp=0.0, + ) + per_invocation.append( + types.SimpleNamespace( + actual_invocation=actual_invocation, + expected_invocation=None, + eval_metric_results=list(metrics), + ) + ) + return types.SimpleNamespace( + eval_set_id="set_a", + eval_id=case_id, + run_id=1, + final_eval_status=status, + error_message=None if status == "PASSED" else "case failed", + overall_eval_metric_results=list(metrics), + eval_metric_result_per_invocation=per_invocation, + session_id=f"session_{case_id}", + user_id="test_user", + session_details=None, + ) + + +def _sdk_evaluate_result(runs_by_case_id: dict[str, list[object]]): + return types.SimpleNamespace( + results_by_eval_set_id={ + "set_a": types.SimpleNamespace( + eval_results_by_eval_id=dict(runs_by_case_id), + num_runs=max((len(runs) for runs in runs_by_case_id.values()), default=1), + ) + } + ) + + +def _install_fake_agent_evaluator( + monkeypatch, + *, + result: object, + on_evaluate, + raise_after_evaluate: bool, +): + calls: dict[str, object] = {} + + class FakeExecuter: + async def evaluate(self): + calls["observed_candidate"] = bool(on_evaluate()) + if raise_after_evaluate: + raise RuntimeError("case failed after producing a structured result") + + def get_result(self): + return result + + class FakeAgentEvaluator: + @staticmethod + def get_executer(dataset_path, **kwargs): + calls["dataset_path"] = dataset_path + calls.update(kwargs) + return FakeExecuter() + + fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") + fake_eval_module.AgentEvaluator = FakeAgentEvaluator + monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) + + call_agent_module = types.ModuleType("fake_call_agent_module") + + async def call_agent(query: str) -> str: + return query + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + return calls + + def _write_sdk_optimizer_config(tmp_path: Path) -> Path: path = tmp_path / "sdk_optimizer.json" path.write_text( From c6b517228e6d5ba711b72116fac6b77b91c9ff26 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 21:00:44 +0800 Subject: [PATCH 20/34] fix(examples): parse sdk evalset expectations --- .../eval_optimize_loop/eval_loop/backends.py | 137 +++++++++++++++- .../tests/test_sdk_backend.py | 154 +++++++++++++++++- 2 files changed, 280 insertions(+), 11 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index ddc5ebca..1aec140d 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -16,6 +16,7 @@ from .fake_judge import FakeJudge from .fake_model import FakeModel from .loader import load_eval_cases +from .loader import read_json from .optimizer import FakeOptimizer from .schemas import CandidatePrompt from .schemas import CaseResult @@ -421,7 +422,7 @@ async def evaluate( target_paths, context=f"cannot evaluate {prompt_id}", ) - expected_cases = load_eval_cases(dataset_path, split=split) + expected_cases = _load_sdk_expected_cases(dataset_path, split=split) snapshot = snapshot_prompt_files(target_paths) result: Any | None = None with temporary_prompt_bundle(snapshot, candidate_prompts): @@ -447,7 +448,7 @@ async def evaluate( result, prompt_id=prompt_id, split=split, - expected_cases=expected_cases, + expected_cases=expected_cases.values(), ) def _load_required_call_agent(self, *, for_evaluation: bool): @@ -721,6 +722,138 @@ def _safe_jsonable(value: Any) -> Any: return repr(value) +def _load_sdk_expected_cases( + dataset_path: str | Path, + *, + split: str, +) -> dict[str, EvalCase]: + """Load wrapper metadata from an SDK EvalSet without changing the SDK file.""" + + payload = read_json(dataset_path) + if "eval_cases" not in payload: + if "cases" in payload: + return _eval_cases_by_id( + load_eval_cases(dataset_path, split=split), + context=f"legacy evalset {dataset_path}", + ) + raise ValueError(f"SDK evalset {dataset_path} must contain an eval_cases list") + + eval_set_id = payload.get("eval_set_id") + if not isinstance(eval_set_id, str) or not eval_set_id.strip(): + raise ValueError(f"SDK evalset {dataset_path} is missing non-empty eval_set_id") + raw_cases = payload["eval_cases"] + if not isinstance(raw_cases, list): + raise ValueError(f"SDK evalset {dataset_path} eval_cases must be a list") + + expected_cases: dict[str, EvalCase] = {} + for index, raw_case in enumerate(raw_cases): + if not isinstance(raw_case, dict): + raise ValueError( + f"SDK evalset {dataset_path} eval_cases[{index}] must be an object" + ) + eval_id = raw_case.get("eval_id") + if not isinstance(eval_id, str) or not eval_id.strip(): + raise ValueError( + f"SDK evalset {dataset_path} eval_cases[{index}] is missing non-empty eval_id" + ) + if eval_id in expected_cases: + raise ValueError(f"SDK evalset {dataset_path} contains duplicate eval_id {eval_id!r}") + expected_cases[eval_id] = _expected_case_from_sdk_case( + raw_case, + eval_id=eval_id, + split=split, + dataset_path=dataset_path, + ) + return expected_cases + + +def _expected_case_from_sdk_case( + raw_case: dict[str, Any], + *, + eval_id: str, + split: str, + dataset_path: str | Path, +) -> EvalCase: + context = f"SDK evalset {dataset_path} case {eval_id!r}" + conversation = raw_case.get("conversation") + if not isinstance(conversation, list) or not conversation: + raise ValueError(f"{context} must contain a non-empty conversation list") + + input_text = "" + for turn_index, invocation in reversed(list(enumerate(conversation))): + if not isinstance(invocation, dict): + raise ValueError(f"{context} conversation[{turn_index}] must be an object") + user_content = invocation.get("user_content") + if user_content is None: + continue + if not isinstance(user_content, dict): + raise ValueError( + f"{context} conversation[{turn_index}].user_content must be an object" + ) + candidate_text = _content_text(user_content) + if candidate_text.strip(): + input_text = candidate_text + break + if not input_text: + raise ValueError(f"{context} conversation has no user_content text") + + session_input = raw_case.get("session_input") + if not isinstance(session_input, dict): + raise ValueError(f"{context} must contain a session_input object") + state = session_input.get("state") + if not isinstance(state, dict): + raise ValueError(f"{context} session_input.state must be an object") + expectation = state.get("eval_optimize_expectation") + if not isinstance(expectation, dict): + raise ValueError( + f"{context} session_input.state must contain eval_optimize_expectation object" + ) + + tags = state.get("eval_optimize_tags", []) + if not isinstance(tags, list) or any(not isinstance(tag, str) for tag in tags): + raise ValueError(f"{context} eval_optimize_tags must be a list of strings") + protected = state.get("eval_optimize_protected", False) + if not isinstance(protected, bool): + raise ValueError(f"{context} eval_optimize_protected must be a boolean") + + expected_failure_category = state.get("eval_optimize_expected_failure_category") + if expected_failure_category is None: + expected_failure_category = state.get("expected_failure_category") + if expected_failure_category is None: + expected_failure_category = expectation.get("expected_failure_category") + if ( + expected_failure_category is not None + and ( + not isinstance(expected_failure_category, str) + or not expected_failure_category.strip() + ) + ): + raise ValueError(f"{context} expected_failure_category must be a non-empty string") + + return EvalCase( + case_id=eval_id, + split=split, + input=input_text, + expectation=dict(expectation), + tags=list(tags), + protected=protected, + expected_failure_category=expected_failure_category, + ) + + +def _eval_cases_by_id( + cases: Iterable[EvalCase], + *, + context: str, +) -> dict[str, EvalCase]: + by_id: dict[str, EvalCase] = {} + for case in cases: + if case.case_id in by_id: + raise ValueError(f"{context} contains duplicate case id {case.case_id!r}") + by_id[case.case_id] = case + return by_id + + def _eval_result_from_sdk_result( result: Any, *, diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 941c6759..e12cdfd1 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -328,6 +328,92 @@ def test_sdk_result_mapping_rejects_non_finite_metric_scores(): ) +def test_sdk_expected_cases_parse_standard_evalset_metadata(tmp_path: Path): + dataset_path = tmp_path / "validation.evalset.json" + case_payload = _sdk_eval_case_payload( + "case-1", + query="query", + expected="expected", + tags=["x"], + protected=True, + ) + case_payload["conversation"].insert( + 0, + { + "invocation_id": "case-1-turn-0", + "user_content": { + "role": "user", + "parts": [{"text": "earlier query"}], + }, + "final_response": { + "role": "model", + "parts": [{"text": "earlier expected"}], + }, + }, + ) + dataset_path.write_text( + json.dumps( + _sdk_evalset_payload([case_payload]) + ), + encoding="utf-8", + ) + + expected_cases = backend_module._load_sdk_expected_cases( + dataset_path, + split="validation", + ) + + assert set(expected_cases) == {"case-1"} + expected_case = expected_cases["case-1"] + assert expected_case.input == "query" + assert expected_case.expectation == { + "type": "exact", + "expected": "expected", + "expected_failure_category": "final_response_mismatch", + } + assert expected_case.tags == ["x"] + assert expected_case.protected is True + assert expected_case.expected_failure_category == "final_response_mismatch" + assert expected_case.split == "validation" + + +def test_sdk_expected_cases_reject_duplicate_eval_ids(tmp_path: Path): + dataset_path = tmp_path / "duplicate.evalset.json" + dataset_path.write_text( + json.dumps( + _sdk_evalset_payload([ + _sdk_eval_case_payload("case-1"), + _sdk_eval_case_payload("case-1"), + ]) + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="duplicate.*case-1"): + backend_module._load_sdk_expected_cases(dataset_path, split="validation") + + +def test_sdk_expected_cases_require_expectation_metadata(tmp_path: Path): + payload = _sdk_evalset_payload([_sdk_eval_case_payload("case-1")]) + del payload["eval_cases"][0]["session_input"]["state"]["eval_optimize_expectation"] + dataset_path = tmp_path / "missing_expectation.evalset.json" + dataset_path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="case-1.*eval_optimize_expectation"): + backend_module._load_sdk_expected_cases(dataset_path, split="validation") + + +def test_sdk_expected_cases_reject_invalid_eval_cases_shape(tmp_path: Path): + dataset_path = tmp_path / "invalid_shape.evalset.json" + dataset_path.write_text( + json.dumps({"eval_set_id": "set", "eval_cases": {}}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="eval_cases.*list"): + backend_module._load_sdk_expected_cases(dataset_path, split="validation") + + @pytest.mark.asyncio async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_bytes( tmp_path: Path, @@ -335,17 +421,21 @@ async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_byt ): dataset_path = tmp_path / "validation.evalset.json" dataset_path.write_text( - json.dumps({ - "split": "validation", - "cases": [{ - "case_id": "case_a", - "input": "question", - "expectation": {"answer": "answer"}, - "expected_failure_category": "format_violation", - }], - }), + json.dumps( + _sdk_evalset_payload([ + _sdk_eval_case_payload( + "case_a", + query="question", + expected="answer", + expected_failure_category="format_violation", + ) + ]) + ), encoding="utf-8", ) + from trpc_agent_sdk.evaluation import EvalSet + + assert EvalSet.model_validate_json(dataset_path.read_text(encoding="utf-8")).eval_cases[0].eval_id == "case_a" prompt_path = tmp_path / "prompt.txt" original_bytes = b"original prompt\r\n" prompt_path.write_bytes(original_bytes) @@ -385,6 +475,7 @@ async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_byt assert calls["eval_result_output_dir"] == str(tmp_path / "sdk_eval") assert prompt_path.read_bytes() == original_bytes assert mapped.cases[0].trace_available is True + assert mapped.cases[0].expected_failure_category == "format_violation" def test_sdk_backend_requires_call_agent_path(tmp_path: Path): @@ -1397,6 +1488,51 @@ def _sdk_evaluate_result(runs_by_case_id: dict[str, list[object]]): ) +def _sdk_evalset_payload(eval_cases: list[dict[str, object]]) -> dict[str, object]: + return { + "eval_set_id": "set", + "eval_cases": eval_cases, + } + + +def _sdk_eval_case_payload( + eval_id: str, + *, + query: str = "query", + expected: str = "expected", + expected_failure_category: str = "final_response_mismatch", + tags: list[str] | None = None, + protected: bool = False, +) -> dict[str, object]: + return { + "eval_id": eval_id, + "conversation": [{ + "invocation_id": f"{eval_id}-turn-1", + "user_content": { + "role": "user", + "parts": [{"text": query}], + }, + "final_response": { + "role": "model", + "parts": [{"text": expected}], + }, + }], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "test-user", + "state": { + "eval_optimize_expectation": { + "type": "exact", + "expected": expected, + "expected_failure_category": expected_failure_category, + }, + "eval_optimize_tags": list(tags or []), + "eval_optimize_protected": protected, + }, + }, + } + + def _install_fake_agent_evaluator( monkeypatch, *, From b91bf05d55d10bc23b766bea29212320393842a3 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 21:31:06 +0800 Subject: [PATCH 21/34] fix(evaluation): make sdk backend integration real --- .../eval_optimize_loop/eval_loop/backends.py | 223 +++++++++- .../tests/test_sdk_backend.py | 418 +++++++++++++++++- tests/evaluation/test_agent_evaluator.py | 65 +++ trpc_agent_sdk/evaluation/__init__.py | 2 + trpc_agent_sdk/evaluation/_agent_evaluator.py | 24 +- 5 files changed, 699 insertions(+), 33 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 1aec140d..ce650871 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -218,10 +218,12 @@ async def optimize_async( """Compatibility async wrapper returning the historical candidate list.""" target_paths = self._target_prompt_paths() - if set(target_paths) == {"system_prompt"}: - baseline_prompts = {"system_prompt": baseline_prompt} - else: + try: baseline_prompts = _read_prompt_bundle(target_paths) + except FileNotFoundError: + # Let optimize_candidates report dependency/import failures before + # it reaches its authoritative source snapshot. + baseline_prompts = {name: baseline_prompt for name in target_paths} result = await self.optimize_candidates( baseline_prompts=baseline_prompts, baseline_train=EvalResult( @@ -260,12 +262,13 @@ async def optimize_candidates( raise ValueError(f"sdk mode could not import AgentOptimizer/TargetPrompt: {exc}") from exc target_paths = self._target_prompt_paths() + snapshot = snapshot_prompt_files(target_paths) + source_bundle = _prompt_bundle_from_snapshot(snapshot) baseline_bundle = _validated_prompt_bundle( baseline_prompts, target_paths, context="baseline prompt bundle", ) - source_bundle = _read_prompt_bundle(target_paths) mismatched = sorted( name for name in target_paths if baseline_bundle[name] != source_bundle[name] ) @@ -279,7 +282,6 @@ async def optimize_candidates( for name, path in target_paths.items(): target_prompt.add_path(name, str(path)) - snapshot = snapshot_prompt_files(target_paths) try: sdk_result = await AgentOptimizer.optimize( config_path=str(config_path), @@ -299,10 +301,27 @@ async def optimize_candidates( + ", ".join(changed_sources) ) - total_llm_cost = _finite_result_field( + _require_successful_optimize_result(sdk_result) + total_llm_cost = _nonnegative_result_field( "total_llm_cost", getattr(sdk_result, "total_llm_cost", 0.0), ) + _pass_rate_result_field( + "baseline_pass_rate", + getattr(sdk_result, "baseline_pass_rate", 0.0), + ) + _pass_rate_result_field( + "best_pass_rate", + getattr(sdk_result, "best_pass_rate", 0.0), + ) + _finite_result_field( + "pass_rate_improvement", + getattr(sdk_result, "pass_rate_improvement", 0.0), + ) + _nonnegative_result_field( + "duration_seconds", + getattr(sdk_result, "duration_seconds", 0.0), + ) best_raw = getattr(sdk_result, "best_prompts", None) if not best_raw: raise ValueError("sdk mode completed but OptimizeResult.best_prompts was empty") @@ -315,8 +334,12 @@ async def optimize_candidates( candidates: list[CandidatePrompt] = [] rounds: list[OptimizationRound] = [] seen_bundles: set[tuple[tuple[str, str], ...]] = set() + seen_round_ids: set[int] = set() for index, sdk_round in enumerate(getattr(sdk_result, "rounds", []) or [], start=1): round_id = _round_id(sdk_round, fallback=index) + if round_id in seen_round_ids: + raise ValueError(f"duplicate SDK round id: {round_id}") + seen_round_ids.add(round_id) candidate_id = f"sdk_round_{round_id:03d}" round_raw_prompts = getattr(sdk_round, "candidate_prompts", {}) or {} round_prompts = ( @@ -333,11 +356,19 @@ async def optimize_candidates( getattr(sdk_round, "metric_breakdown", {}) or {}, context=f"SDK round {round_id} metric_breakdown", ) - round_cost_value = _finite_number( + _pass_rate_number( + getattr(sdk_round, "train_pass_rate", 0.0), + context=f"SDK round {round_id} train_pass_rate", + ) + _pass_rate_number( + getattr(sdk_round, "validation_pass_rate", 0.0), + context=f"SDK round {round_id} validation_pass_rate", + ) + round_cost_value = _nonnegative_number( getattr(sdk_round, "round_llm_cost", 0.0), context=f"SDK round {round_id} round_llm_cost", ) - duration_seconds = _finite_number( + duration_seconds = _nonnegative_number( getattr(sdk_round, "duration_seconds", 0.0), context=f"SDK round {round_id} duration_seconds", ) @@ -413,6 +444,8 @@ async def evaluate( call_agent = self._load_required_call_agent(for_evaluation=True) try: from trpc_agent_sdk.evaluation import AgentEvaluator + from trpc_agent_sdk.evaluation import EvalConfig + from trpc_agent_sdk.evaluation import EvaluationCasesFailed except Exception as exc: # pragma: no cover - depends on optional SDK import health raise ValueError(f"sdk mode could not import AgentEvaluator: {exc}") from exc @@ -423,6 +456,15 @@ async def evaluate( context=f"cannot evaluate {prompt_id}", ) expected_cases = _load_sdk_expected_cases(dataset_path, split=split) + artifact_path = Path(artifact_dir) + artifact_path.mkdir(parents=True, exist_ok=True) + eval_config_path = artifact_path / "eval_config.json" + eval_config_path.write_text( + EvalConfig( + criteria={"final_response_avg_score": 1.0} + ).model_dump_json(indent=2), + encoding="utf-8", + ) snapshot = snapshot_prompt_files(target_paths) result: Any | None = None with temporary_prompt_bundle(snapshot, candidate_prompts): @@ -432,10 +474,11 @@ async def evaluate( print_detailed_results=False, print_summary_report=False, eval_result_output_dir=str(artifact_dir), + eval_metrics_file_path_or_dir=str(eval_config_path), ) try: await executer.evaluate() - except Exception: + except EvaluationCasesFailed: result = executer.get_result() if result is None: raise @@ -551,10 +594,28 @@ def _validated_prompt_bundle( def _read_prompt_bundle(paths: dict[str, str | Path]) -> dict[str, str]: - return { - name: Path(path).read_text(encoding="utf-8") - for name, path in paths.items() - } + prompts: dict[str, str] = {} + for name, path in paths.items(): + prompt_path = Path(path) + try: + prompts[name] = prompt_path.read_bytes().decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError( + f"source prompt field {name!r} is not valid UTF-8: {prompt_path}" + ) from exc + return prompts + + +def _prompt_bundle_from_snapshot(snapshot: Any) -> dict[str, str]: + prompts: dict[str, str] = {} + for name, prompt_file in snapshot.files.items(): + try: + prompts[name] = prompt_file.content.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError( + f"source prompt field {name!r} is not valid UTF-8: {prompt_file.path}" + ) from exc + return prompts def _changed_snapshot_files(snapshot: Any) -> list[str]: @@ -577,6 +638,30 @@ def _round_id(round_record: Any, *, fallback: int) -> int: return value +def _require_successful_optimize_result(result: Any) -> None: + status = _sdk_result_text(getattr(result, "status", None)).upper() + if status == "SUCCEEDED": + return + raise ValueError( + "SDK optimization did not succeed: " + f"status={status}; " + f"error_message={_sdk_result_text(getattr(result, 'error_message', None))}; " + f"finish_reason={_sdk_result_text(getattr(result, 'finish_reason', None))}; " + f"stop_reason={_sdk_result_text(getattr(result, 'stop_reason', None))}" + ) + + +def _sdk_result_text(value: Any) -> str: + enum_value = getattr(value, "value", None) + if enum_value is not None: + value = enum_value + else: + enum_name = getattr(value, "name", None) + if enum_name is not None: + value = enum_name + return str(value).split(".")[-1] + + def _prompt_bundle_key(prompts: dict[str, str]) -> tuple[tuple[str, str], ...]: return tuple(sorted(prompts.items())) @@ -609,11 +694,11 @@ def _summarize_sdk_result(result: Any) -> dict[str, Any]: "finish_reason": _safe_jsonable(getattr(result, "finish_reason", None)), "stop_reason": _safe_jsonable(getattr(result, "stop_reason", None)), "error_message": _safe_jsonable(getattr(result, "error_message", None)), - "baseline_pass_rate": _finite_result_field( + "baseline_pass_rate": _pass_rate_result_field( "baseline_pass_rate", getattr(result, "baseline_pass_rate", 0.0), ), - "best_pass_rate": _finite_result_field( + "best_pass_rate": _pass_rate_result_field( "best_pass_rate", getattr(result, "best_pass_rate", 0.0), ), @@ -636,12 +721,12 @@ def _summarize_sdk_result(result: Any) -> dict[str, Any]: "per_metric_best_candidates": _safe_jsonable( getattr(result, "per_metric_best_candidates", {}) ), - "total_llm_cost": _finite_result_field( + "total_llm_cost": _nonnegative_result_field( "total_llm_cost", getattr(result, "total_llm_cost", 0.0), ), "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})), - "duration_seconds": _finite_result_field( + "duration_seconds": _nonnegative_result_field( "duration_seconds", getattr(result, "duration_seconds", 0.0), ), @@ -686,6 +771,22 @@ def _finite_result_field(field_name: str, value: Any) -> float: raise ValueError(f"SDK OptimizeResult field {field_name} must be a finite number") from exc +def _nonnegative_result_field(field_name: str, value: Any) -> float: + number = _finite_result_field(field_name, value) + if number < 0.0: + raise ValueError(f"SDK OptimizeResult field {field_name} must be non-negative") + return number + + +def _pass_rate_result_field(field_name: str, value: Any) -> float: + number = _finite_result_field(field_name, value) + if not 0.0 <= number <= 1.0: + raise ValueError( + f"SDK OptimizeResult field {field_name} must be between 0 and 1" + ) + return number + + def _finite_metric_map(value: Any, *, context: str) -> dict[str, float]: if not isinstance(value, dict): raise ValueError(f"{context} must be a metric mapping") @@ -704,6 +805,20 @@ def _finite_number(value: Any, *, context: str) -> float: return number +def _nonnegative_number(value: Any, *, context: str) -> float: + number = _finite_number(value, context=context) + if number < 0.0: + raise ValueError(f"{context} must be non-negative") + return number + + +def _pass_rate_number(value: Any, *, context: str) -> float: + number = _finite_number(value, context=context) + if not 0.0 <= number <= 1.0: + raise ValueError(f"{context} must be between 0 and 1") + return number + + def _safe_jsonable(value: Any) -> Any: if hasattr(value, "model_dump"): return _safe_jsonable(value.model_dump(mode="json")) @@ -744,6 +859,8 @@ def _load_sdk_expected_cases( raw_cases = payload["eval_cases"] if not isinstance(raw_cases, list): raise ValueError(f"SDK evalset {dataset_path} eval_cases must be a list") + if not raw_cases: + raise ValueError(f"SDK evalset {dataset_path} eval_cases must not be empty") expected_cases: dict[str, EvalCase] = {} for index, raw_case in enumerate(raw_cases): @@ -872,13 +989,29 @@ def _eval_result_from_sdk_result( sdk_runs_by_id: dict[str, list[Any]] = {} results_by_eval_set_id = getattr(result, "results_by_eval_set_id", {}) or {} - for set_result in results_by_eval_set_id.values(): + if not results_by_eval_set_id: + raise ValueError("SDK EvaluateResult contains no eval set results") + for raw_eval_set_id, set_result in results_by_eval_set_id.items(): + eval_set_id = str(raw_eval_set_id) + num_runs = _optional_num_runs(set_result, eval_set_id=eval_set_id) eval_results_by_eval_id = getattr(set_result, "eval_results_by_eval_id", {}) or {} for raw_eval_id, runs in eval_results_by_eval_id.items(): eval_id = str(raw_eval_id) if eval_id in sdk_runs_by_id: raise ValueError(f"SDK evaluation result contains duplicate case id: {eval_id}") - sdk_runs_by_id[eval_id] = list(runs or []) + run_list = list(runs or []) + if num_runs is not None and len(run_list) != num_runs: + raise ValueError( + f"SDK eval set {eval_set_id!r} declares num_runs={num_runs}, " + f"but case {eval_id!r} contains {len(run_list)} runs" + ) + _validate_sdk_run_ids( + run_list, + eval_set_id=eval_set_id, + eval_id=eval_id, + num_runs=num_runs, + ) + sdk_runs_by_id[eval_id] = run_list expected_ids = set(expected_by_id) sdk_ids = set(sdk_runs_by_id) @@ -966,6 +1099,58 @@ def _eval_result_from_sdk_result( ) +def _optional_num_runs(set_result: Any, *, eval_set_id: str) -> int | None: + if not hasattr(set_result, "num_runs"): + return None + num_runs = getattr(set_result, "num_runs") + if isinstance(num_runs, bool) or not isinstance(num_runs, int) or num_runs <= 0: + raise ValueError( + f"SDK eval set {eval_set_id!r} num_runs must be a positive integer" + ) + return num_runs + + +def _validate_sdk_run_ids( + runs: list[Any], + *, + eval_set_id: str, + eval_id: str, + num_runs: int | None, +) -> None: + seen_run_ids: set[int] = set() + for run in runs: + internal_eval_id = getattr(run, "eval_id", None) + if internal_eval_id not in (None, "") and str(internal_eval_id) != eval_id: + raise ValueError( + f"SDK run internal eval_id {internal_eval_id!r} does not match " + f"container case id {eval_id!r}" + ) + internal_eval_set_id = getattr(run, "eval_set_id", None) + if ( + internal_eval_set_id not in (None, "") + and str(internal_eval_set_id) != eval_set_id + ): + raise ValueError( + f"SDK run internal eval_set_id {internal_eval_set_id!r} does not match " + f"container eval set id {eval_set_id!r}" + ) + + run_id = getattr(run, "run_id", None) + if run_id is None: + continue + if isinstance(run_id, bool) or not isinstance(run_id, int) or run_id <= 0: + raise ValueError( + f"SDK case {eval_id!r} run_id must be a positive integer or None" + ) + if num_runs is not None and run_id > num_runs: + raise ValueError( + f"SDK case {eval_id!r} run_id {run_id} exceeds num_runs={num_runs}" + ) + if run_id in seen_run_ids: + raise ValueError(f"SDK evaluation result contains duplicate run_id {run_id} for case {eval_id}") + seen_run_ids.add(run_id) + + def _aggregate_case_metrics(runs: list[Any], *, case_id: str) -> dict[str, float]: scores_by_metric: dict[str, list[float]] = {} for run in runs: diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index e12cdfd1..8faa3c65 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -212,6 +212,167 @@ async def test_sdk_backend_appends_best_when_no_round_contains_it(tmp_path: Path assert result.candidates[1].prompt_fields == best_prompts +@pytest.mark.asyncio +async def test_sdk_backend_rejects_duplicate_round_ids(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="best prompt", + rounds=[ + _sdk_round(1, {"system_prompt": "first"}), + _sdk_round(1, {"system_prompt": "second"}), + ], + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="duplicate SDK round id: 1"): + await SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ["FAILED", "CANCELED"]) +async def test_sdk_backend_rejects_unsuccessful_optimize_result( + tmp_path: Path, + monkeypatch, + status: str, +): + from trpc_agent_sdk.evaluation import OptimizeResult + + sdk_result = OptimizeResult( + algorithm="gepa_reflective", + status=status, + finish_reason="error", + stop_reason="user_requested_stop", + error_message="optimizer exploded", + baseline_pass_rate=0.5, + best_pass_rate=0.5, + pass_rate_improvement=0.0, + baseline_prompts={"system_prompt": "baseline"}, + best_prompts={"system_prompt": "baseline"}, + total_rounds=0, + rounds=[], + total_reflection_lm_calls=0, + total_llm_cost=0.0, + duration_seconds=0.1, + started_at="2026-07-04T12:00:00+00:00", + finished_at="2026-07-04T12:00:00.100000+00:00", + ) + _install_fake_sdk(monkeypatch, result_override=sdk_result) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ) + + with pytest.raises(ValueError) as exc_info: + await backend.optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + message = str(exc_info.value) + assert f"status={status}" in message + assert "error_message=optimizer exploded" in message + assert "finish_reason=error" in message + assert "stop_reason=user_requested_stop" in message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field_name", "field_value", "message"), + [ + ("total_llm_cost", -0.01, "total_llm_cost.*non-negative"), + ("duration_seconds", -0.01, "duration_seconds.*non-negative"), + ("baseline_pass_rate", -0.01, "baseline_pass_rate.*between 0 and 1"), + ("best_pass_rate", 1.01, "best_pass_rate.*between 0 and 1"), + ], +) +async def test_sdk_backend_rejects_invalid_top_level_numeric_values( + tmp_path: Path, + monkeypatch, + field_name, + field_value, + message, +): + kwargs = {field_name: field_value, "rounds": []} + _install_fake_sdk(monkeypatch, best_prompt="best", **kwargs) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match=message): + await SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field_name", "field_value", "message"), + [ + ("round_llm_cost", -0.01, "round_llm_cost.*non-negative"), + ("duration_seconds", -0.01, "duration_seconds.*non-negative"), + ("train_pass_rate", -0.01, "train_pass_rate.*between 0 and 1"), + ("validation_pass_rate", 1.01, "validation_pass_rate.*between 0 and 1"), + ], +) +async def test_sdk_backend_rejects_invalid_round_numeric_values( + tmp_path: Path, + monkeypatch, + field_name, + field_value, + message, +): + round_record = _sdk_round(1, {"system_prompt": "candidate"}) + setattr(round_record, field_name, field_value) + _install_fake_sdk( + monkeypatch, + best_prompt="best", + rounds=[round_record], + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match=message): + await SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): expected_case = EvalCase( case_id="case_a", @@ -223,6 +384,7 @@ def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): first_run = _sdk_case_run( "case_a", status="FAILED", + run_id=1, metrics=[ _sdk_metric("response_match", 0.5, status="FAILED", reason="wrong response"), _sdk_metric("style", 0.25, status="PASSED"), @@ -234,6 +396,7 @@ def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): last_run = _sdk_case_run( "case_a", status="FAILED", + run_id=2, metrics=[ _sdk_metric("response_match", 1.0, status="PASSED"), _sdk_metric("style", 0.75, status="PASSED"), @@ -328,6 +491,74 @@ def test_sdk_result_mapping_rejects_non_finite_metric_scores(): ) +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ("eval_id", "wrong_case", "internal eval_id.*wrong_case.*case_a"), + ("eval_set_id", "wrong_set", "internal eval_set_id.*wrong_set.*set_a"), + ], +) +def test_sdk_result_mapping_rejects_internal_run_identity_mismatch( + attribute, + value, + message, +): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + run = _sdk_case_run("case_a", status="PASSED", metrics=[]) + setattr(run, attribute, value) + + with pytest.raises(ValueError, match=message): + backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result({"case_a": [run]}), + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_result_mapping_rejects_duplicate_run_ids(): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + runs = [ + _sdk_case_run("case_a", status="PASSED", metrics=[], run_id=1), + _sdk_case_run("case_a", status="PASSED", metrics=[], run_id=1), + ] + + with pytest.raises(ValueError, match="duplicate run_id 1.*case_a"): + backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result({"case_a": runs}), + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_result_mapping_rejects_num_runs_mismatch(): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + run = _sdk_case_run("case_a", status="PASSED", metrics=[]) + result = _sdk_evaluate_result({"case_a": [run]}) + result.results_by_eval_set_id["set_a"].num_runs = 2 + + with pytest.raises(ValueError, match="num_runs=2.*case_a.*1"): + backend_module._eval_result_from_sdk_result( + result, + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_result_mapping_rejects_empty_evaluate_result(): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + + with pytest.raises(ValueError, match="EvaluateResult contains no eval set results"): + backend_module._eval_result_from_sdk_result( + types.SimpleNamespace(results_by_eval_set_id={}), + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + def test_sdk_expected_cases_parse_standard_evalset_metadata(tmp_path: Path): dataset_path = tmp_path / "validation.evalset.json" case_payload = _sdk_eval_case_payload( @@ -393,6 +624,17 @@ def test_sdk_expected_cases_reject_duplicate_eval_ids(tmp_path: Path): backend_module._load_sdk_expected_cases(dataset_path, split="validation") +def test_sdk_expected_cases_reject_empty_standard_evalset(tmp_path: Path): + dataset_path = tmp_path / "empty.evalset.json" + dataset_path.write_text( + json.dumps(_sdk_evalset_payload([])), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="eval_cases must not be empty"): + backend_module._load_sdk_expected_cases(dataset_path, split="validation") + + def test_sdk_expected_cases_require_expectation_metadata(tmp_path: Path): payload = _sdk_evalset_payload([_sdk_eval_case_payload("case-1")]) del payload["eval_cases"][0]["session_input"]["state"]["eval_optimize_expectation"] @@ -419,6 +661,9 @@ async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_byt tmp_path: Path, monkeypatch, ): + from trpc_agent_sdk.evaluation import EvalConfig + from trpc_agent_sdk.evaluation import EvaluationCasesFailed + dataset_path = tmp_path / "validation.evalset.json" dataset_path.write_text( json.dumps( @@ -455,7 +700,7 @@ async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_byt monkeypatch, result=sdk_result, on_evaluate=lambda: prompt_path.read_text(encoding="utf-8") == "candidate prompt", - raise_after_evaluate=True, + evaluation_error=EvaluationCasesFailed("case failed"), ) backend = SDKBackend( prompt_path=prompt_path, @@ -473,11 +718,126 @@ async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_byt assert calls["observed_candidate"] is True assert calls["eval_result_output_dir"] == str(tmp_path / "sdk_eval") + eval_config_path = Path(calls["eval_metrics_file_path_or_dir"]) + eval_config = EvalConfig.model_validate_json( + eval_config_path.read_text(encoding="utf-8") + ) + assert eval_config.criteria == {"final_response_avg_score": 1.0} + assert [metric.metric_name for metric in eval_config.get_eval_metrics()] == [ + "final_response_avg_score" + ] assert prompt_path.read_bytes() == original_bytes assert mapped.cases[0].trace_available is True assert mapped.cases[0].expected_failure_category == "format_violation" +@pytest.mark.asyncio +async def test_sdk_backend_evaluate_propagates_non_case_failure_even_with_result( + tmp_path: Path, + monkeypatch, +): + dataset_path = tmp_path / "validation.evalset.json" + dataset_path.write_text( + json.dumps( + _sdk_evalset_payload([ + _sdk_eval_case_payload( + "case_a", + query="question", + expected="answer", + ) + ]) + ), + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + original_bytes = b"original prompt\r\n" + prompt_path.write_bytes(original_bytes) + sdk_result = _sdk_evaluate_result({ + "case_a": [ + _sdk_case_run( + "case_a", + status="PASSED", + metrics=[_sdk_metric("quality", 1.0, status="PASSED")], + output="answer", + user_content="question", + ) + ] + }) + _install_fake_agent_evaluator( + monkeypatch, + result=sdk_result, + on_evaluate=lambda: True, + evaluation_error=RuntimeError("network failed"), + ) + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ) + + with pytest.raises(RuntimeError, match="network failed"): + await backend.evaluate( + prompt_id="candidate", + prompts={"system_prompt": "candidate prompt"}, + dataset_path=dataset_path, + split="validation", + trace=False, + artifact_dir=tmp_path / "sdk_eval", + ) + assert prompt_path.read_bytes() == original_bytes + + +@pytest.mark.asyncio +async def test_sdk_backend_evaluate_real_agent_evaluator_smoke( + tmp_path: Path, + monkeypatch, +): + dataset_path = (tmp_path / "validation.evalset.json").resolve() + dataset_path.write_text( + json.dumps( + _sdk_evalset_payload([ + _sdk_eval_case_payload( + "case_a", + query="question", + expected="answer", + ) + ]) + ), + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("original prompt", encoding="utf-8") + calls: list[str] = [] + call_agent_module = types.ModuleType("real_sdk_smoke_call_agent") + + async def call_agent(query: str) -> str: + calls.append(query) + return "answer" + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "real_sdk_smoke_call_agent", call_agent_module) + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="real_sdk_smoke_call_agent:call_agent", + ) + + result = await backend.evaluate( + prompt_id="candidate", + prompts={"system_prompt": "candidate prompt"}, + dataset_path=dataset_path, + split="validation", + trace=False, + artifact_dir=tmp_path / "real_sdk_eval", + ) + + assert calls == ["question"] + assert len(result.cases) == 1 + assert result.cases[0].case_id == "case_a" + assert result.cases[0].passed is True + assert result.cases[0].metrics == {"final_response_avg_score": 1.0} + assert result.cases[0].output == "answer" + assert prompt_path.read_text(encoding="utf-8") == "original prompt" + + def test_sdk_backend_requires_call_agent_path(tmp_path: Path): backend = SDKBackend(prompt_path=tmp_path / "prompt.txt") @@ -645,6 +1005,45 @@ def test_sdk_backend_never_delegates_source_writeback(tmp_path: Path, monkeypatc assert "update_source" not in vars(backend) +@pytest.mark.asyncio +async def test_sdk_backend_detects_source_change_at_snapshot_boundary( + tmp_path: Path, + monkeypatch, +): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + real_snapshot = backend_module.snapshot_prompt_files + + def snapshot_after_external_change(paths): + prompt_path.write_text("external change", encoding="utf-8") + return real_snapshot(paths) + + monkeypatch.setattr( + backend_module, + "snapshot_prompt_files", + snapshot_after_external_change, + ) + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ) + + with pytest.raises(ValueError, match="baseline prompt bundle.*source"): + await backend.optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + assert "update_source" not in calls + assert prompt_path.read_text(encoding="utf-8") == "external change" + + def test_sdk_backend_call_agent_import_failure_names_target(tmp_path: Path): backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="missing.module:call_agent") @@ -1311,6 +1710,7 @@ def _install_fake_sdk( started_at: str | None = None, rounds: list[object] | None = None, write_source_when_requested: bool = False, + result_override: object | None = None, ): calls = {} @@ -1326,6 +1726,8 @@ class FakeAgentOptimizer: @staticmethod async def optimize(**kwargs): calls.update(kwargs) + if result_override is not None: + return result_override if write_source_when_requested and kwargs.get("update_source"): for _, path in kwargs["target_prompt"].paths: Path(path).write_bytes(b"optimizer mutated source") @@ -1443,6 +1845,7 @@ def _sdk_case_run( *, status: str, metrics: list[object], + run_id: int | None = 1, output: str | None = None, user_content: str | None = None, intermediate_data: object | None = None, @@ -1466,7 +1869,7 @@ def _sdk_case_run( return types.SimpleNamespace( eval_set_id="set_a", eval_id=case_id, - run_id=1, + run_id=run_id, final_eval_status=status, error_message=None if status == "PASSED" else "case failed", overall_eval_metric_results=list(metrics), @@ -1538,15 +1941,15 @@ def _install_fake_agent_evaluator( *, result: object, on_evaluate, - raise_after_evaluate: bool, + evaluation_error: BaseException | None, ): calls: dict[str, object] = {} class FakeExecuter: async def evaluate(self): calls["observed_candidate"] = bool(on_evaluate()) - if raise_after_evaluate: - raise RuntimeError("case failed after producing a structured result") + if evaluation_error is not None: + raise evaluation_error def get_result(self): return result @@ -1558,8 +1961,13 @@ def get_executer(dataset_path, **kwargs): calls.update(kwargs) return FakeExecuter() + from trpc_agent_sdk.evaluation._agent_evaluator import _EvaluationCasesFailed + from trpc_agent_sdk.evaluation._eval_config import EvalConfig + fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") fake_eval_module.AgentEvaluator = FakeAgentEvaluator + fake_eval_module.EvalConfig = EvalConfig + fake_eval_module.EvaluationCasesFailed = _EvaluationCasesFailed monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) call_agent_module = types.ModuleType("fake_call_agent_module") diff --git a/tests/evaluation/test_agent_evaluator.py b/tests/evaluation/test_agent_evaluator.py index 8609ccc5..81324149 100644 --- a/tests/evaluation/test_agent_evaluator.py +++ b/tests/evaluation/test_agent_evaluator.py @@ -5,10 +5,13 @@ # tRPC-Agent-Python is licensed under Apache-2.0. """Unit tests for agent evaluator (agent_evaluator).""" +import json + import pytest import trpc_agent_sdk.runners # noqa: F401 +from trpc_agent_sdk.evaluation import _agent_evaluator as agent_evaluator_module from trpc_agent_sdk.evaluation import EvalStatus from trpc_agent_sdk.evaluation import EvalCaseResult from trpc_agent_sdk.evaluation import EvalMetricResult @@ -18,6 +21,68 @@ from trpc_agent_sdk.evaluation import PassNC +def test_evaluation_cases_failed_is_public_with_private_alias(): + from trpc_agent_sdk import evaluation + from trpc_agent_sdk.evaluation._agent_evaluator import _EvaluationCasesFailed + + assert evaluation.EvaluationCasesFailed is _EvaluationCasesFailed + assert issubclass(evaluation.EvaluationCasesFailed, AssertionError) + + +@pytest.mark.parametrize( + ("raw_path", "expected"), + [ + (r"C:\x\a.evalset.json", (r"C:\x\a.evalset.json", None)), + ( + r"C:\x\a.evalset.json:case-1", + (r"C:\x\a.evalset.json", "case-1"), + ), + ( + "/tmp/a.evalset.json:case-1", + ("/tmp/a.evalset.json", "case-1"), + ), + ], +) +def test_split_eval_set_selector_is_drive_safe(raw_path, expected): + assert agent_evaluator_module._split_eval_set_selector(raw_path) == expected + + +def test_load_eval_set_from_absolute_path_and_selector(tmp_path): + evalset_path = (tmp_path / "sample.evalset.json").resolve() + evalset_path.write_text( + json.dumps({ + "eval_set_id": "set", + "eval_cases": [{ + "eval_id": "case-1", + "conversation": [{ + "invocation_id": "turn-1", + "user_content": { + "role": "user", + "parts": [{"text": "query"}], + }, + "final_response": { + "role": "model", + "parts": [{"text": "expected"}], + }, + }], + }], + }), + encoding="utf-8", + ) + + loaded = AgentEvaluator._load_eval_set_from_file( + str(evalset_path), + agent_evaluator_module.EvalConfig(criteria={}), + ) + selected = AgentEvaluator._load_eval_set_from_file( + f"{evalset_path}:case-1", + agent_evaluator_module.EvalConfig(criteria={}), + ) + + assert [case.eval_id for case in loaded.eval_cases] == ["case-1"] + assert [case.eval_id for case in selected.eval_cases] == ["case-1"] + + class TestPassNC: """Test suite for PassNC dataclass.""" diff --git a/trpc_agent_sdk/evaluation/__init__.py b/trpc_agent_sdk/evaluation/__init__.py index 8f614b4b..b70ac64a 100644 --- a/trpc_agent_sdk/evaluation/__init__.py +++ b/trpc_agent_sdk/evaluation/__init__.py @@ -34,6 +34,7 @@ # Main evaluation entry point from ._agent_evaluator import AgentEvaluator +from ._agent_evaluator import EvaluationCasesFailed from ._agent_evaluator import PassNC from ._common import EvalBaseModel from ._criterion_registry import CRITERION_REGISTRY @@ -220,6 +221,7 @@ "DEFAULT_OPTIMIZE_MAX_TOKENS", "DEFAULT_OPTIMIZE_TEMPERATURE", "EvaluationOutcome", + "EvaluationCasesFailed", "FinishReason", "FrameworkStopConfig", "GepaReflectiveAlgo", diff --git a/trpc_agent_sdk/evaluation/_agent_evaluator.py b/trpc_agent_sdk/evaluation/_agent_evaluator.py index a2e47009..8f5a0641 100644 --- a/trpc_agent_sdk/evaluation/_agent_evaluator.py +++ b/trpc_agent_sdk/evaluation/_agent_evaluator.py @@ -75,7 +75,7 @@ _RESULT_HANDLER = _utils.EvalResultHandler() -class _EvaluationCasesFailed(AssertionError): +class EvaluationCasesFailed(AssertionError): """Signal raised by ``_EvalExecuter._run`` when one or more eval cases fail. Subclasses :class:`AssertionError` so direct ``AgentEvaluator.evaluate`` @@ -93,6 +93,19 @@ class _EvaluationCasesFailed(AssertionError): """ +_EvaluationCasesFailed = EvaluationCasesFailed + + +def _split_eval_set_selector(eval_set_file: str) -> tuple[str, Optional[str]]: + """Split an optional .json:case_id suffix without consuming a drive.""" + + drive, tail = os.path.splitdrive(eval_set_file) + json_path, separator, case_id = tail.rpartition(":") + if not separator or not case_id or not json_path.lower().endswith(".json"): + return (eval_set_file, None) + return (drive + json_path, case_id) + + @dataclass(frozen=True) class PassNC: """(n, c): n = runs, c = runs that all passed (for pass@k / pass^k).""" @@ -660,14 +673,7 @@ def _load_eval_set_from_file( FileNotFoundError: If file doesn't exist ValueError: If file format is invalid or eval case not found """ - # Check if file_path contains a case selector (ADK style: "file.json:case_id") - selected_case_id = None - actual_file_path = eval_set_file - - if ":" in eval_set_file: - parts = eval_set_file.split(":", 1) - actual_file_path = parts[0] - selected_case_id = parts[1] + actual_file_path, selected_case_id = _split_eval_set_selector(eval_set_file) if not os.path.exists(actual_file_path): raise FileNotFoundError(f"Eval set file not found: {actual_file_path}") From 6e42d292371680a890e0a58d33b038f328c6d6c5 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Fri, 10 Jul 2026 21:51:43 +0800 Subject: [PATCH 22/34] fix(examples): remove fake evaluation oracle leakage --- .../data/train.evalset.json | 185 +++++--- .../eval_optimize_loop/data/val.evalset.json | 184 +++++--- .../eval_optimize_loop/eval_loop/backends.py | 38 +- .../eval_loop/fake_model.py | 183 ++++---- .../eval_optimize_loop/eval_loop/optimizer.py | 67 ++- .../tests/test_fake_model_generalization.py | 400 +++++++++++++++--- .../test_no_sample_case_id_hardcoding.py | 149 ++----- .../tests/test_sdk_backend.py | 39 +- 8 files changed, 835 insertions(+), 410 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json index 70580991..9b162057 100644 --- a/examples/optimization/eval_optimize_loop/data/train.evalset.json +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -1,59 +1,144 @@ { - "evalset_id": "eval_optimize_loop_train_v1", - "split": "train", - "description": "Training cases used for failure attribution and fake prompt optimization.", - "cases": [ + "eval_set_id": "eval_optimize_loop_train_v1", + "name": "Evaluation optimization loop train set", + "description": "Training cases used for failure attribution and prompt optimization.", + "eval_cases": [ { - "id": "train_json_refund", - "input": "Return strict JSON for a refund escalation. Use keys intent and priority.", - "expectation": { - "type": "json", - "required_keys": [ - "intent", - "priority" - ], - "expected_values": { - "intent": "refund", - "priority": "high" - }, - "expected_failure_category": "format_violation" - }, - "tags": [ - "json", - "format" - ] + "eval_id": "train_json_refund", + "conversation": [ + { + "invocation_id": "train_json_refund_turn_1", + "user_content": { + "parts": [ + { + "text": "Return strict JSON with intent=refund and priority=high." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"intent\":\"refund\",\"priority\":\"high\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "trainer", + "state": { + "eval_optimize_expectation": { + "type": "json", + "required_keys": [ + "intent", + "priority" + ], + "expected_values": { + "intent": "refund", + "priority": "high" + }, + "expected_failure_category": "format_violation" + }, + "eval_optimize_tags": [ + "json", + "format" + ] + } + } }, { - "id": "train_exact_order_status", - "input": "Answer exactly READY when the order can ship.", - "expectation": { - "type": "exact", - "expected": "READY", - "expected_failure_category": "final_response_mismatch" - }, - "tags": [ - "exact", - "format" - ] + "eval_id": "train_exact_order_status", + "conversation": [ + { + "invocation_id": "train_json_order_turn_1", + "user_content": { + "parts": [ + { + "text": "Return strict JSON with status=READY and next_step=ship." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"next_step\":\"ship\",\"status\":\"READY\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "trainer", + "state": { + "eval_optimize_expectation": { + "type": "json", + "required_keys": [ + "status", + "next_step" + ], + "expected_values": { + "status": "READY", + "next_step": "ship" + }, + "expected_failure_category": "format_violation" + }, + "eval_optimize_tags": [ + "json", + "format" + ] + } + } }, { - "id": "train_rubric_retry_summary", - "input": "In 80 chars or less, mention latency and retries.", - "expectation": { - "type": "rubric", - "must_include": [ - "latency", - "retries" - ], - "forbidden": [ - "database" - ], - "max_chars": 80 - }, - "tags": [ - "rubric", - "no_effect" - ] + "eval_id": "train_rubric_retry_summary", + "conversation": [ + { + "invocation_id": "train_latency_retries_turn_1", + "user_content": { + "parts": [ + { + "text": "Explain latency and retries naturally in under 80 characters." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "Latency can trigger retries." + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "trainer", + "state": { + "eval_optimize_expectation": { + "type": "rubric", + "must_include": [ + "latency", + "retries" + ], + "forbidden": [ + "database" + ], + "max_chars": 80 + }, + "eval_optimize_tags": [ + "rubric", + "no_effect" + ] + } + } } ] } diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json index ceeaf11d..9262b14f 100644 --- a/examples/optimization/eval_optimize_loop/data/val.evalset.json +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -1,63 +1,141 @@ { - "evalset_id": "eval_optimize_loop_validation_v1", - "split": "validation", + "eval_set_id": "eval_optimize_loop_validation_v1", + "name": "Evaluation optimization loop validation set", "description": "Validation cases that separate safe optimization from train-set overfitting.", - "cases": [ + "eval_cases": [ { - "id": "val_json_invoice", - "input": "Return strict JSON for an invoice approval. Use keys status and next_step.", - "expectation": { - "type": "json", - "required_keys": [ - "status", - "next_step" - ], - "expected_values": { - "next_step": "email_customer", - "status": "approved" - }, - "expected_failure_category": "format_violation" - }, - "tags": [ - "json", - "safe_improves" - ] + "eval_id": "val_json_invoice", + "conversation": [ + { + "invocation_id": "val_json_approval_turn_1", + "user_content": { + "parts": [ + { + "text": "Return strict JSON with status=approved and next_step=email_customer." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"next_step\":\"email_customer\",\"status\":\"approved\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "validator", + "state": { + "eval_optimize_expectation": { + "type": "json", + "required_keys": [ + "status", + "next_step" + ], + "expected_values": { + "next_step": "email_customer", + "status": "approved" + }, + "expected_failure_category": "format_violation" + }, + "eval_optimize_tags": [ + "json", + "safe_improves" + ] + } + } }, { - "id": "val_explain_cache", - "input": "Explain cache invalidation for a junior engineer in natural language.", - "expectation": { - "type": "rubric", - "must_include": [ - "cache", - "stale data" - ], - "forbidden": [ - "{", - "}", - "json" - ], - "max_chars": 180, - "expected_failure_category": "format_violation" - }, - "tags": [ - "rubric", - "overfit_guard" - ] + "eval_id": "val_explain_cache", + "conversation": [ + { + "invocation_id": "val_cache_stale_turn_1", + "user_content": { + "parts": [ + { + "text": "Explain cache invalidation and stale data naturally." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "Cache invalidation decides when cached data is stale data and must be refreshed." + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "validator", + "state": { + "eval_optimize_expectation": { + "type": "rubric", + "must_include": [ + "cache", + "stale data" + ], + "forbidden": [ + "{", + "}", + "json" + ], + "max_chars": 180, + "expected_failure_category": "format_violation" + }, + "eval_optimize_tags": [ + "rubric", + "overfit_guard" + ] + } + } }, { - "id": "val_protected_yes_no", - "input": "Answer exactly YES if idempotent retries are safe for duplicate requests.", - "expectation": { - "type": "exact", - "expected": "YES", - "expected_failure_category": "final_response_mismatch" - }, - "protected": true, - "tags": [ - "protected", - "exact" - ] + "eval_id": "val_protected_yes_no", + "conversation": [ + { + "invocation_id": "val_return_yes_turn_1", + "user_content": { + "parts": [ + { + "text": "Return only YES; do not use JSON." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "YES" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "validator", + "state": { + "eval_optimize_expectation": { + "type": "exact", + "expected": "YES", + "expected_failure_category": "final_response_mismatch" + }, + "eval_optimize_tags": [ + "protected", + "exact" + ], + "eval_optimize_protected": true + } + } } ] } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index ce650871..5b3263a2 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -7,6 +7,7 @@ import math from collections.abc import Iterable from dataclasses import dataclass +from dataclasses import replace from pathlib import Path from typing import Any from typing import Protocol @@ -84,18 +85,33 @@ async def evaluate( ) -> EvalResult: del artifact_dir prompt = _required_system_prompt(prompts, context=f"cannot evaluate {prompt_id}") - cases = load_eval_cases(dataset_path, split=split) + cases = _load_sdk_expected_cases(dataset_path, split=split).values() evaluator = ExampleEvaluator( FakeModel(seed=self.seed), FakeJudge(), trace_enabled=trace, ) - return evaluator.evaluate( + result = evaluator.evaluate( prompt_id=prompt_id, prompt=prompt, cases=cases, split=split, ) + if not trace: + return result + + sanitized_cases = [] + for case_result in result.cases: + model_trace = case_result.trace.get("model") + sanitized_trace = dict(model_trace) if isinstance(model_trace, dict) else {} + sanitized_cases.append( + replace( + case_result, + trace=sanitized_trace, + trace_available=bool(sanitized_trace), + ) + ) + return replace(result, cases=sanitized_cases) async def optimize_candidates( self, @@ -113,7 +129,13 @@ async def optimize_candidates( baseline_prompts, context="cannot optimize fake prompt bundle", ) - candidates = _normalize_fake_candidates(self._optimizer.propose(baseline_prompt)) + candidates = _normalize_fake_candidates( + self._optimizer.propose( + baseline_prompt, + baseline_train, + failure_summary, + ) + ) zero_cost = CostSummary(complete=True) rounds = [ OptimizationRound( @@ -147,10 +169,14 @@ def optimize( optimizer_config_path: str | Path, output_dir: str | Path, ) -> list[CandidatePrompt]: - """Compatibility wrapper for the pre-async fake pipeline.""" + """Reject legacy optimization that has no observed failure evidence.""" - del train_path, val_path, optimizer_config_path, output_dir - return _normalize_fake_candidates(self._optimizer.propose(baseline_prompt)) + del baseline_prompt, train_path, val_path, optimizer_config_path, output_dir + raise RuntimeError( + "FakeBackend.optimize() cannot invent training failure evidence; " + "use await FakeBackend.optimize_candidates(...) with baseline_train " + "and failure_summary." + ) class SDKBackend: diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py index a7555e74..f031ffdf 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py @@ -3,135 +3,102 @@ from __future__ import annotations import json +import re +from dataclasses import dataclass from typing import Any from .schemas import EvalCase +_ASSIGNMENT_PATTERN = re.compile(r"(? None: self.seed = seed - def generate(self, prompt_id: str, prompt: str, case: EvalCase) -> tuple[str, dict[str, Any], float]: + def generate( + self, + prompt_id: str, + prompt: str, + case: EvalCase, + ) -> tuple[str, dict[str, Any], float]: mode = self._mode(prompt) - output_override = self._simulated_output(case, mode) - if output_override is not None: - output = output_override - elif mode == "overfit": - output = self._overfit_output(case) - elif mode == "safe": - output = self._safe_output(case) - else: - output = self._baseline_output(case) + request = self._parse_request(case.input) + output = self._render(request, mode=mode) trace = { "seed": self.seed, "prompt_id": prompt_id, "prompt_mode": mode, - "case_id": case.case_id, - "expectation_type": case.expectation.get("type"), } return output, trace, self.COST_PER_CALL - def _mode(self, prompt: str) -> str: - if "ALWAYS_OUTPUT_JSON" in prompt: + @staticmethod + def _mode(prompt: str) -> str: + if _OVERFIT_INSTRUCTION in prompt: return "overfit" - if "STRICT_WHEN_REQUESTED" in prompt: + if _SAFE_INSTRUCTION in prompt: return "safe" return "baseline" - def _baseline_output(self, case: EvalCase) -> str: - expectation_type = case.expectation.get("type") - if expectation_type == "json": - return f"Here is the JSON you requested: {self._expected_json(case)}" - if expectation_type == "exact": - expected = str(case.expectation.get("expected", "")) - if case.protected or "baseline_pass" in case.tags: - return expected - return f"{expected} - confirmed." - if expectation_type == "rubric": - return self._rubric_sentence(case) - if expectation_type == "tool": - return self._tool_json(case) - if expectation_type == "knowledge": - return self._knowledge_sentence(case) - return self._rubric_sentence(case) - - def _overfit_output(self, case: EvalCase) -> str: - expectation_type = case.expectation.get("type") - if case.split == "train": - return self._ideal_output(case) - if expectation_type == "json": - return self._expected_json(case) - if expectation_type == "exact": - return json.dumps({"answer": str(case.expectation.get("expected", ""))}, sort_keys=True) - if expectation_type == "rubric" or "prose" in case.tags: - return json.dumps({"answer": self._rubric_sentence(case)}, sort_keys=True) - if expectation_type == "tool": - return self._tool_json(case) - if expectation_type == "knowledge": - return self._knowledge_sentence(case) - return json.dumps({"answer": self._rubric_sentence(case)}, sort_keys=True) - - def _safe_output(self, case: EvalCase) -> str: - user_asked = case.input.lower() - expectation_type = case.expectation.get("type") - if expectation_type == "json" or "json" in user_asked: - return self._expected_json(case) - if expectation_type == "exact" or "exactly" in user_asked: - return str(case.expectation.get("expected", "")) - return self._ideal_output(case) - - def _ideal_output(self, case: EvalCase) -> str: - expectation_type = case.expectation.get("type") - if expectation_type == "json": - return self._expected_json(case) - if expectation_type == "exact": - return str(case.expectation.get("expected", "")) - if expectation_type == "rubric": - return self._rubric_sentence(case) - if expectation_type == "tool": - return self._tool_json(case) - if expectation_type == "knowledge": - return self._knowledge_sentence(case) - return self._rubric_sentence(case) - - def _expected_json(self, case: EvalCase) -> str: - values = dict(case.expectation.get("expected_values") or {}) - return json.dumps(values, sort_keys=True) - - def _simulated_output(self, case: EvalCase, mode: str) -> str | None: - return case.simulated_outputs.get(mode) - - def _rubric_sentence(self, case: EvalCase) -> str: - must_include = [str(item) for item in case.expectation.get("must_include") or []] - if must_include: - sentence = " ".join(must_include) - else: - sentence = "The answer satisfies the rubric" - max_chars = case.expectation.get("max_chars") - if max_chars is not None and len(sentence) > int(max_chars): - sentence = sentence[:int(max_chars)].rstrip() - return sentence - - def _tool_json(self, case: EvalCase) -> str: - tool_name = str(case.expectation.get("expected_tool", "lookup")) - args = dict(case.expectation.get("expected_args") or {}) - return json.dumps({"tool": tool_name, "args": args}, sort_keys=True) - - def _knowledge_sentence(self, case: EvalCase) -> str: - terms = [str(item) for item in case.expectation.get("must_include_knowledge_terms") or []] - sources = [str(item) for item in case.expectation.get("required_sources") or []] - parts = terms + sources - return " ".join(parts) if parts else "knowledge source recalled" + @staticmethod + def _parse_request(user_input: str) -> _ParsedRequest: + assignments = {match.group(1): match.group(2) for match in _ASSIGNMENT_PATTERN.finditer(user_input)} + only_match = _RETURN_ONLY_PATTERN.search(user_input) + return _ParsedRequest( + assignments=assignments, + only_value=only_match.group(1) if only_match else None, + strict_json=bool(_STRICT_JSON_PATTERN.search(user_input)), + natural_answer=FakeModel._natural_answer(user_input), + ) + + @staticmethod + def _natural_answer(user_input: str) -> str: + lowered = user_input.lower() + if "latency" in lowered and "retries" in lowered: + return "Latency can trigger retries." + if "cache" in lowered and "stale data" in lowered: + return "Cache invalidation refreshes stale data." + return "Here is a natural response." + + @staticmethod + def _render(request: _ParsedRequest, *, mode: str) -> str: + if mode == "overfit": + payload: dict[str, str] + if request.assignments: + payload = request.assignments + else: + payload = { + "answer": request.only_value or request.natural_answer, + } + return json.dumps(payload, sort_keys=True) + + if request.strict_json: + payload_json = json.dumps(request.assignments, sort_keys=True) + if mode == "safe": + return payload_json + return f"Here is the JSON you requested: {payload_json}" + + if request.only_value is not None: + return request.only_value + + return request.natural_answer diff --git a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py index da91ee12..f0990f01 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py @@ -4,35 +4,50 @@ from .diffing import make_unified_diff from .schemas import CandidatePrompt +from .schemas import EvalResult + +_TARGET_FAILURE_CATEGORIES = { + "format_violation", + "final_response_mismatch", +} +_OVERFIT_INSTRUCTION = "Always force every final answer into JSON." +_SAFE_INSTRUCTION = "Use strict JSON only when the user explicitly asks." class FakeOptimizer: - """Produce the two candidates required by the example issue.""" - - def propose(self, baseline_prompt: str) -> list[CandidatePrompt]: - overfit_prompt = ( - baseline_prompt.rstrip() - + "\n\n" - + "# Optimizer patch\n" - + "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n" - + "Always force every final answer into JSON, even when the user asks for prose.\n" - ) - safe_prompt = ( - baseline_prompt.rstrip() - + "\n\n" - + "# Optimizer patch\n" - + "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n" - + "Use strict JSON only when the user explicitly asks for JSON.\n" - + "Use exact answers only when the user explicitly asks for an exact answer.\n" - + "Otherwise answer naturally and honor rubric constraints.\n" - ) + """Propose candidates only for observed train formatting failures.""" + + def propose( + self, + baseline_prompt: str, + baseline_train: EvalResult, + failure_summary: dict[str, object], + ) -> list[CandidatePrompt]: + failed_cases = [case for case in baseline_train.cases if not case.passed] + if not failed_cases: + return [] + + observed_categories = {case.failure_category for case in failed_cases if case.failure_category} + by_category = failure_summary.get("by_category") + if isinstance(by_category, dict): + observed_categories.update( + str(category) for category, count in by_category.items() if _is_positive_count(count) + ) + + targeted = sorted(observed_categories & _TARGET_FAILURE_CATEGORIES) + if not targeted: + return [] + + overfit_prompt = f"{baseline_prompt.rstrip()}\n\n{_OVERFIT_INSTRUCTION}\n" + safe_prompt = f"{baseline_prompt.rstrip()}\n\n{_SAFE_INSTRUCTION}\n" + evidence = ", ".join(targeted) return [ CandidatePrompt( candidate_id="candidate_001_overfit", prompt=overfit_prompt, rationale=( - "The train failures are strict JSON/exact formatting failures, so this candidate " - "over-corrects by forcing JSON globally." + f"Observed training failures ({evidence}); this candidate deliberately " + "tests the risky global-JSON correction." ), prompt_diff=make_unified_diff( baseline_prompt, @@ -40,13 +55,14 @@ def propose(self, baseline_prompt: str) -> list[CandidatePrompt]: before_name="baseline_system_prompt.txt", after_name="candidate_001_overfit/system_prompt.txt", ), + prompt_fields={"system_prompt": overfit_prompt}, ), CandidatePrompt( candidate_id="candidate_002_safe", prompt=safe_prompt, rationale=( - "This candidate fixes observed strict-format failures without changing " - "natural-language behavior on validation cases." + f"Observed training failures ({evidence}); this candidate limits strict " + "JSON behavior to explicit user requests." ), prompt_diff=make_unified_diff( baseline_prompt, @@ -54,5 +70,10 @@ def propose(self, baseline_prompt: str) -> list[CandidatePrompt]: before_name="baseline_system_prompt.txt", after_name="candidate_002_safe/system_prompt.txt", ), + prompt_fields={"system_prompt": safe_prompt}, ), ] + + +def _is_positive_count(value: object) -> bool: + return not isinstance(value, bool) and isinstance(value, (int, float)) and value > 0 diff --git a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py index 5bd26f9a..57f16e05 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py +++ b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py @@ -1,77 +1,375 @@ from __future__ import annotations import json +from dataclasses import replace +from pathlib import Path -from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge +import pytest +from trpc_agent_sdk.evaluation import EvalSet + +from examples.optimization.eval_optimize_loop.eval_loop.backends import FakeBackend from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.optimizer import FakeOptimizer +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult + +ROOT = Path(__file__).resolve().parents[1] +TRAIN_PATH = ROOT / "data" / "train.evalset.json" +VALIDATION_PATH = ROOT / "data" / "val.evalset.json" +BASELINE_PROMPT = "Answer the user's request." +OVERFIT_INSTRUCTION = "Always force every final answer into JSON" +SAFE_INSTRUCTION = "Use strict JSON only when the user explicitly asks" +OVERFIT_PROMPT = f"{BASELINE_PROMPT}\n\n{OVERFIT_INSTRUCTION}." +SAFE_PROMPT = f"{BASELINE_PROMPT}\n\n{SAFE_INSTRUCTION}." -def test_fake_model_json_behavior_does_not_depend_on_case_id(): - case = EvalCase( - case_id="hidden_json_case", - split="validation", - input="Return JSON.", - expectation={"type": "json", "expected_values": {"status": "ok"}, "required_keys": ["status"]}, + +def test_model_output_and_trace_ignore_all_evaluator_only_case_fields(): + input_text = "Return strict JSON with intent=refund and priority=high." + base = EvalCase( + case_id="oracle_case_a", + split="train", + input=input_text, + expectation={ + "type": "json", + "expected_values": {"secret": "first-oracle"}, + }, + tags=["first-tag"], + protected=False, + simulated_outputs={"safe": "FIRST SIMULATED ORACLE"}, ) + variants = [ + base, + replace(base, case_id="oracle_case_b"), + replace( + base, + expectation={"type": "exact", "expected": "SECOND ORACLE"}, + ), + replace(base, split="validation"), + replace(base, protected=True), + replace(base, tags=["different", "oracle-tag"]), + replace(base, simulated_outputs={"safe": "SECOND SIMULATED ORACLE"}), + ] + + generated = [FakeModel(seed=91).generate("candidate", SAFE_PROMPT, case) for case in variants] + + assert {output for output, _, _ in generated} == {'{"intent": "refund", "priority": "high"}'} + assert {json.dumps(trace, sort_keys=True) for _, trace, _ in generated} == { + json.dumps( + {"seed": 91, "prompt_id": "candidate", "prompt_mode": "safe"}, + sort_keys=True, + ) + } + serialized_traces = json.dumps([trace for _, trace, _ in generated]) + assert "ORACLE" not in serialized_traces + assert "oracle_case" not in serialized_traces + + +def test_model_is_deterministic_and_prompt_id_or_seed_cannot_change_output(): + case = _case("Return strict JSON with status=READY and next_step=ship.") model = FakeModel(seed=91) - baseline, _, _ = model.generate("baseline", "plain prompt", case) - safe, _, _ = model.generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) + first = model.generate("first-id", SAFE_PROMPT, case) + repeated = model.generate("first-id", SAFE_PROMPT, case) + different_prompt_id = model.generate("second-id", SAFE_PROMPT, case) + different_seed = FakeModel(seed=999).generate("first-id", SAFE_PROMPT, case) + + assert first == repeated + assert first[0] == different_prompt_id[0] == different_seed[0] + assert first[0] == '{"next_step": "ship", "status": "READY"}' + + +@pytest.mark.parametrize( + ("prompt", "expected"), + [ + ( + BASELINE_PROMPT, + 'Here is the JSON you requested: {"intent": "refund", "priority": "high"}', + ), + (OVERFIT_PROMPT, '{"intent": "refund", "priority": "high"}'), + (SAFE_PROMPT, '{"intent": "refund", "priority": "high"}'), + ], +) +def test_strict_json_rendering_uses_assignments_from_user_input( + prompt: str, + expected: str, +): + case = _case("Return StRiCt JsOn with intent=refund and priority=high.") - assert baseline.startswith("Here is the JSON") - assert json.loads(safe) == {"status": "ok"} - assert FakeJudge().score(case, baseline).passed is False - assert FakeJudge().score(case, safe).passed is True + output, _, _ = FakeModel(seed=91).generate("arbitrary-id", prompt, case) + assert output == expected -def test_fake_model_protected_exact_baseline_passes_but_overfit_regresses(): - case = EvalCase( - case_id="hidden_protected_case", - split="validation", - input="Answer exactly YES.", - expectation={"type": "exact", "expected": "YES"}, - protected=True, + +@pytest.mark.parametrize( + ("prompt", "expected"), + [ + (BASELINE_PROMPT, "YES"), + (SAFE_PROMPT, "YES"), + (OVERFIT_PROMPT, '{"answer": "YES"}'), + ], +) +def test_return_only_rendering_is_prompt_mode_sensitive(prompt: str, expected: str): + case = _case("ReTuRn OnLy YES; do not use JSON.") + + output, _, _ = FakeModel(seed=91).generate("arbitrary-id", prompt, case) + + assert output == expected + + +@pytest.mark.parametrize( + ("input_text", "natural"), + [ + ( + "Explain latency and retries naturally in under 80 characters.", + "Latency can trigger retries.", + ), + ( + "Explain cache invalidation and stale data naturally.", + "Cache invalidation refreshes stale data.", + ), + ("Explain an unknown topic naturally.", "Here is a natural response."), + ], +) +def test_natural_and_unknown_requests_are_deterministic(input_text: str, natural: str): + case = _case(input_text) + + baseline, _, _ = FakeModel(seed=91).generate("baseline", BASELINE_PROMPT, case) + safe, _, _ = FakeModel(seed=91).generate("safe", SAFE_PROMPT, case) + overfit, _, _ = FakeModel(seed=91).generate("overfit", OVERFIT_PROMPT, case) + + assert baseline == safe == natural + assert json.loads(overfit) == {"answer": natural} + + +def test_optimizer_returns_no_candidates_without_observed_target_failures(): + optimizer = FakeOptimizer() + all_pass = _eval_result( + _case_result( + "passed_case", + passed=True, + failure_category="format_violation", + ) + ) + unrelated_failure = _eval_result( + _case_result( + "failed_case", + passed=False, + failure_category="llm_rubric_not_met", + ) + ) + + assert ( + optimizer.propose( + BASELINE_PROMPT, + all_pass, + {"by_category": {"format_violation": 1}}, + ) + == [] + ) + assert ( + optimizer.propose( + BASELINE_PROMPT, + unrelated_failure, + {"by_category": {"llm_rubric_not_met": 1}}, + ) + == [] ) - model = FakeModel(seed=91) - baseline, _, _ = model.generate("baseline", "plain prompt", case) - overfit, _, _ = model.generate("overfit", "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON", case) - safe, _, _ = model.generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) - assert baseline == "YES" - assert overfit == '{"answer": "YES"}' - assert safe == "YES" +@pytest.mark.parametrize( + ("case_failure_category", "failure_summary"), + [ + ("format_violation", {}), + (None, {"by_category": {"final_response_mismatch": 2}}), + ], +) +def test_optimizer_proposes_natural_language_candidates_from_observed_failures( + case_failure_category: str | None, + failure_summary: dict[str, object], +): + baseline_train = _eval_result( + _case_result( + "random_training_case", + passed=False, + failure_category=case_failure_category, + ) + ) + candidates = FakeOptimizer().propose( + BASELINE_PROMPT, + baseline_train, + failure_summary, + ) + assert [candidate.candidate_id for candidate in candidates] == [ + "candidate_001_overfit", + "candidate_002_safe", + ] + assert OVERFIT_INSTRUCTION in candidates[0].prompt + assert SAFE_INSTRUCTION in candidates[1].prompt + assert all("OPTIMIZER_MARKER" not in candidate.prompt for candidate in candidates) + assert all(candidate.prompt_fields == {"system_prompt": candidate.prompt} for candidate in candidates) + assert all(candidate.rationale for candidate in candidates) + assert all(candidate.prompt_diff for candidate in candidates) -def test_fake_model_rubric_overfit_forces_json_on_validation_prose(): - case = EvalCase( - case_id="hidden_rubric_case", - split="validation", - input="Explain in prose.", - expectation={ - "type": "rubric", - "must_include": ["cache", "stale data"], - "forbidden": ["{", "}", "json"], - "max_chars": 120, - }, - tags=["prose"], + +@pytest.mark.asyncio +async def test_fake_backend_passes_training_evidence_to_optimizer(tmp_path: Path): + backend = FakeBackend(seed=91) + no_evidence = await backend.optimize_candidates( + baseline_prompts={"system_prompt": BASELINE_PROMPT}, + baseline_train=_eval_result(), + failure_summary={}, + train_path=TRAIN_PATH, + validation_path=VALIDATION_PATH, + config_path=tmp_path / "unused.json", + artifact_dir=tmp_path / "unused", + ) + with_evidence = await backend.optimize_candidates( + baseline_prompts={"system_prompt": BASELINE_PROMPT}, + baseline_train=_eval_result( + _case_result( + "not-a-sample-id", + passed=False, + failure_category="format_violation", + ) + ), + failure_summary={"by_category": {"format_violation": 1}}, + train_path=TRAIN_PATH, + validation_path=VALIDATION_PATH, + config_path=tmp_path / "unused.json", + artifact_dir=tmp_path / "unused", ) - output, _, _ = FakeModel(seed=91).generate("overfit", "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON", case) - assert output.startswith("{") - judged = FakeJudge().score(case, output) - assert judged.error_code == "forbidden_pattern" + assert no_evidence.candidates == [] + assert no_evidence.rounds == [] + assert len(with_evidence.candidates) == 2 + assert len(with_evidence.rounds) == 2 + + +def test_fake_backend_sync_optimizer_refuses_to_invent_failure_evidence(tmp_path: Path): + with pytest.raises(RuntimeError, match="failure evidence|optimize_candidates"): + FakeBackend(seed=91).optimize( + baseline_prompt=BASELINE_PROMPT, + train_path=TRAIN_PATH, + val_path=VALIDATION_PATH, + optimizer_config_path=tmp_path / "unused.json", + output_dir=tmp_path / "unused", + ) + + +def test_example_data_is_official_sdk_evalset_with_self_contained_user_inputs(): + expected_inputs = { + TRAIN_PATH: [ + "Return strict JSON with intent=refund and priority=high.", + "Return strict JSON with status=READY and next_step=ship.", + "Explain latency and retries naturally in under 80 characters.", + ], + VALIDATION_PATH: [ + "Return strict JSON with status=approved and next_step=email_customer.", + "Explain cache invalidation and stale data naturally.", + "Return only YES; do not use JSON.", + ], + } + all_eval_ids: list[str] = [] + all_invocation_ids: list[str] = [] + for path, inputs in expected_inputs.items(): + raw = path.read_text(encoding="utf-8") + validated = EvalSet.model_validate_json(raw) + payload = json.loads(raw) + assert [ + case["conversation"][-1]["user_content"]["parts"][0]["text"] for case in payload["eval_cases"] + ] == inputs + assert len(validated.eval_cases) == 3 + all_eval_ids.extend(case.eval_id for case in validated.eval_cases) + all_invocation_ids.extend( + invocation.invocation_id for case in validated.eval_cases for invocation in case.conversation or [] + ) -def test_fake_model_respects_simulated_outputs_override(): - case = EvalCase( - case_id="override_case", + assert len(all_eval_ids) == len(set(all_eval_ids)) == 6 + assert len(all_invocation_ids) == len(set(all_invocation_ids)) == 6 + + +@pytest.mark.asyncio +async def test_fake_backend_scores_baseline_overfit_and_safe_on_official_data( + tmp_path: Path, +): + backend = FakeBackend(seed=91) + prompts = { + "baseline": BASELINE_PROMPT, + "overfit": OVERFIT_PROMPT, + "safe": SAFE_PROMPT, + } + expected = { + ("baseline", "train"): (1 / 3, [False, False, True]), + ("overfit", "train"): (1.0, [True, True, True]), + ("safe", "train"): (1.0, [True, True, True]), + ("baseline", "validation"): (2 / 3, [False, True, True]), + ("overfit", "validation"): (1 / 3, [True, False, False]), + ("safe", "validation"): (1.0, [True, True, True]), + } + + for prompt_id, prompt in prompts.items(): + for split, dataset_path in ( + ("train", TRAIN_PATH), + ("validation", VALIDATION_PATH), + ): + result = await backend.evaluate( + prompt_id=prompt_id, + prompts={"system_prompt": prompt}, + dataset_path=dataset_path, + split=split, + trace=True, + artifact_dir=tmp_path / prompt_id / split, + ) + expected_score, expected_passes = expected[(prompt_id, split)] + assert result.score == pytest.approx(expected_score, abs=1e-6) + assert [case.passed for case in result.cases] == expected_passes + assert all( + case.trace + == { + "seed": 91, + "prompt_id": prompt_id, + "prompt_mode": prompt_id, + } + for case in result.cases + ) + + +def _case(input_text: str) -> EvalCase: + return EvalCase( + case_id="arbitrary-case-id", + split="train", + input=input_text, + expectation={"type": "exact", "expected": "EVALUATOR ORACLE"}, + ) + + +def _case_result( + case_id: str, + *, + passed: bool, + failure_category: str | None, +) -> CaseResult: + return CaseResult( + case_id=case_id, split="train", - input="Any", - expectation={"type": "exact", "expected": "OK"}, - simulated_outputs={"safe": "CUSTOM"}, + score=1.0 if passed else 0.0, + passed=passed, + output="output", + failure_category=failure_category, ) - output, _, _ = FakeModel(seed=91).generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) - assert output == "CUSTOM" + +def _eval_result(*cases: CaseResult) -> EvalResult: + score = sum(case.score for case in cases) / len(cases) if cases else 0.0 + return EvalResult( + prompt_id="baseline", + split="train", + score=score, + passed=all(case.passed for case in cases), + cost=0.0, + cases=list(cases), + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py index 318dd460..16ad5fc6 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py +++ b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py @@ -3,131 +3,46 @@ import json from pathlib import Path -from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline - - -BUSINESS_LOGIC_FILES = [ - "eval_loop/fake_model.py", - "eval_loop/fake_judge.py", - "eval_loop/optimizer.py", - "eval_loop/report.py", - "eval_loop/gate.py", +ROOT = Path(__file__).resolve().parents[1] +DATA_FILES = [ + ROOT / "data" / "train.evalset.json", + ROOT / "data" / "val.evalset.json", +] +NO_SAMPLE_ID_FILES = [ + ROOT / "eval_loop" / "fake_model.py", + ROOT / "eval_loop" / "optimizer.py", + ROOT / "eval_loop" / "backends.py", +] +FORBIDDEN_MODEL_ACCESSES = [ + ".expectation", + ".split", + ".protected", + ".tags", + ".simulated_outputs", + ".case_id", ] -def test_sample_case_ids_do_not_appear_in_business_logic(): - root = Path("examples/optimization/eval_optimize_loop") - case_ids = set() - for rel in ("data/train.evalset.json", "data/val.evalset.json"): - payload = json.loads((root / rel).read_text(encoding="utf-8")) - case_ids.update(case["id"] for case in payload["cases"]) +def test_sample_case_ids_do_not_appear_in_fake_runtime_logic(): + case_ids = { + case["eval_id"] for path in DATA_FILES for case in json.loads(path.read_text(encoding="utf-8"))["eval_cases"] + } - for rel in BUSINESS_LOGIC_FILES: - source = (root / rel).read_text(encoding="utf-8") + for path in NO_SAMPLE_ID_FILES: + source = path.read_text(encoding="utf-8") leaked = sorted(case_id for case_id in case_ids if case_id in source) - assert leaked == [], f"{rel} contains sample case ids: {leaked}" - + assert leaked == [], f"{path.name} contains sample case ids: {leaked}" -def test_uuid_style_case_ids_still_drive_expected_behavior(tmp_path: Path): - train_path = tmp_path / "train.evalset.json" - val_path = tmp_path / "val.evalset.json" - optimizer_path = tmp_path / "optimizer.json" - prompt_path = tmp_path / "prompt.txt" - train_path.write_text( - json.dumps({ - "split": "train", - "cases": [ - _json_case("1d5b548a-39ec-451d-9b01-7d73101b95b1", "train"), - _exact_case("c28596bf-370d-4898-bdf7-e4cc306fa4d3", "train", protected=False), - _rubric_case("3aa39806-878e-4634-9a68-d7f479ab7c9d", "train"), - ], - }), - encoding="utf-8", - ) - val_path.write_text( - json.dumps({ - "split": "validation", - "cases": [ - _json_case("534f045c-06f8-4b96-af55-cb7b6712cc0c", "validation"), - _rubric_case("b6914390-b2bb-4f43-a35b-24b6fe2825cd", "validation"), - _exact_case("70c59d31-adf5-409f-a457-286bcb887f52", "validation", protected=True), - ], - }), - encoding="utf-8", - ) - optimizer_path.write_text( - json.dumps({ - "seed": 91, - "optimizer": {"name": "fake_two_candidate_optimizer"}, - "metrics": {"case_score": "mean"}, - "gate": { - "min_val_score_improvement": 0.01, - "allow_new_hard_fail": False, - "protected_case_ids": ["70c59d31-adf5-409f-a457-286bcb887f52"], - "max_score_drop_per_case": 0.0, - "max_total_cost": 1.0, - }, - }), - encoding="utf-8", - ) - prompt_path.write_text("Baseline prompt", encoding="utf-8") +def test_fake_model_source_cannot_read_evaluator_only_case_fields(): + source = (ROOT / "eval_loop" / "fake_model.py").read_text(encoding="utf-8") - report = run_pipeline( - train_path=train_path, - val_path=val_path, - optimizer_config_path=optimizer_path, - prompt_path=prompt_path, - output_dir=tmp_path / "out", - mode="fake", - trace=True, - ) + leaked = [token for token in FORBIDDEN_MODEL_ACCESSES if token in source] - decisions = {decision.candidate_id: decision for decision in report.gate_decisions} - assert report.selected_candidate == "candidate_002_safe" - assert decisions["candidate_001_overfit"].accepted is False - assert decisions["candidate_001_overfit"].overfit_detected is True - assert decisions["candidate_002_safe"].accepted is True + assert leaked == [] -def _json_case(case_id: str, split: str) -> dict: - return { - "id": case_id, - "input": "Return strict JSON.", - "expectation": { - "type": "json", - "required_keys": ["answer"], - "expected_values": {"answer": "ok"}, - "expected_failure_category": "format_violation", - }, - } - - -def _exact_case(case_id: str, split: str, *, protected: bool) -> dict: - payload = { - "id": case_id, - "input": "Answer exactly YES.", - "expectation": { - "type": "exact", - "expected": "YES", - "expected_failure_category": "final_response_mismatch", - }, - "protected": protected, - "tags": ["baseline_pass"] if protected else [], - } - return payload - - -def _rubric_case(case_id: str, split: str) -> dict: - return { - "id": case_id, - "input": "Explain in prose.", - "expectation": { - "type": "rubric", - "must_include": ["cache", "stale data"], - "forbidden": ["{", "}", "json"], - "max_chars": 120, - "expected_failure_category": "format_violation", - }, - "tags": ["prose"], - } +def test_fake_runtime_uses_no_private_optimizer_markers(): + for relative in ("eval_loop/fake_model.py", "eval_loop/optimizer.py"): + source = (ROOT / relative).read_text(encoding="utf-8") + assert "OPTIMIZER_MARKER" not in source diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 8faa3c65..8d657061 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -11,6 +11,7 @@ from examples.optimization.eval_optimize_loop.eval_loop import backends as backend_module from examples.optimization.eval_optimize_loop.eval_loop.backends import SDKBackend +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationResult @@ -89,12 +90,28 @@ async def test_fake_backend_implements_unified_contract_with_real_trace(tmp_path @pytest.mark.asyncio async def test_fake_backend_wraps_candidates_in_complete_optimization_result(tmp_path: Path): baseline_prompt = DEFAULT_PROMPT.read_text(encoding="utf-8") - baseline_train = _empty_eval_result("baseline", "train") + baseline_train = EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[ + CaseResult( + case_id="observed_training_failure", + split="train", + score=0.0, + passed=False, + output="not-json", + failure_category="format_violation", + ) + ], + ) result = await backend_module.FakeBackend(seed=91).optimize_candidates( baseline_prompts={"system_prompt": baseline_prompt}, baseline_train=baseline_train, - failure_summary={"failed_case_ids": ["train_json_strict"]}, + failure_summary={"by_category": {"format_violation": 1}}, train_path=DEFAULT_TRAIN, validation_path=DEFAULT_VAL, config_path=DEFAULT_OPTIMIZER_CONFIG, @@ -109,6 +126,24 @@ async def test_fake_backend_wraps_candidates_in_complete_optimization_result(tmp assert all(candidate.bundle() == {"system_prompt": candidate.prompt} for candidate in result.candidates) +@pytest.mark.asyncio +async def test_fake_backend_returns_no_candidates_without_failure_categories(tmp_path: Path): + baseline_prompt = DEFAULT_PROMPT.read_text(encoding="utf-8") + + result = await backend_module.FakeBackend(seed=91).optimize_candidates( + baseline_prompts={"system_prompt": baseline_prompt}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={"failed_case_ids": ["unclassified_training_failure"]}, + train_path=DEFAULT_TRAIN, + validation_path=DEFAULT_VAL, + config_path=DEFAULT_OPTIMIZER_CONFIG, + artifact_dir=tmp_path / "fake_optimize_without_categories", + ) + + assert result.candidates == [] + assert result.rounds == [] + + @pytest.mark.asyncio async def test_sdk_backend_maps_rounds_deduplicates_bundles_and_marks_cost_incomplete( tmp_path: Path, From 50a629477feae9b7e70af58763607cc131732745 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 00:24:34 +0800 Subject: [PATCH 23/34] fix(examples): isolate fake model training evidence --- .../eval_optimize_loop/eval_loop/evaluator.py | 2 +- .../eval_loop/fake_model.py | 9 +- .../eval_optimize_loop/eval_loop/optimizer.py | 65 +++- .../tests/test_fake_model_generalization.py | 281 ++++++++++++++---- 4 files changed, 281 insertions(+), 76 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py index 4bd0a615..89e0b2e5 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py @@ -29,7 +29,7 @@ def __init__(self, model: FakeModel, judge: FakeJudge, *, trace_enabled: bool = def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: case_results: list[CaseResult] = [] for case in cases: - output, model_trace, cost = self.model.generate(prompt_id, prompt, case) + output, model_trace, cost = self.model.generate(prompt_id, prompt, case.input) judged = self.judge.score(case, output) failure_category = None failure_reason = None diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py index f031ffdf..f23d56ce 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py @@ -7,8 +7,6 @@ from dataclasses import dataclass from typing import Any -from .schemas import EvalCase - _ASSIGNMENT_PATTERN = re.compile(r"(? tuple[str, dict[str, Any], float]: + if not isinstance(user_input, str): + raise TypeError("user_input must be a string") + mode = self._mode(prompt) - request = self._parse_request(case.input) + request = self._parse_request(user_input) output = self._render(request, mode=mode) trace = { "seed": self.seed, diff --git a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py index f0990f01..64f32947 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections import Counter + from .diffing import make_unified_diff from .schemas import CandidatePrompt from .schemas import EvalResult @@ -23,18 +25,30 @@ def propose( baseline_train: EvalResult, failure_summary: dict[str, object], ) -> list[CandidatePrompt]: - failed_cases = [case for case in baseline_train.cases if not case.passed] - if not failed_cases: - return [] + if not isinstance(failure_summary, dict): + raise TypeError("failure_summary must be a dict") - observed_categories = {case.failure_category for case in failed_cases if case.failure_category} - by_category = failure_summary.get("by_category") - if isinstance(by_category, dict): - observed_categories.update( - str(category) for category, count in by_category.items() if _is_positive_count(count) + if baseline_train.split != "train": + raise ValueError( + "baseline_train.split must be 'train'; " + f"got {baseline_train.split!r}" ) + for case in baseline_train.cases: + if case.split != "train": + raise ValueError( + f"baseline_train case {case.case_id!r} split must be 'train'; " + f"got {case.split!r}" + ) - targeted = sorted(observed_categories & _TARGET_FAILURE_CATEGORIES) + failed_cases = [case for case in baseline_train.cases if not case.passed] + observed_counts = Counter( + case.failure_category + for case in failed_cases + if case.failure_category + ) + _validate_failure_summary(failure_summary, observed_counts) + + targeted = sorted(observed_counts.keys() & _TARGET_FAILURE_CATEGORIES) if not targeted: return [] @@ -75,5 +89,34 @@ def propose( ] -def _is_positive_count(value: object) -> bool: - return not isinstance(value, bool) and isinstance(value, (int, float)) and value > 0 +def _validate_failure_summary( + failure_summary: dict[str, object], + observed_counts: Counter[str], +) -> None: + if "by_category" not in failure_summary: + return + + by_category = failure_summary["by_category"] + if not isinstance(by_category, dict): + raise ValueError("failure_summary['by_category'] must be a dict") + + summary_counts: dict[object, int] = {} + for category, count in by_category.items(): + summary_counts[category] = _normalize_positive_count(category, count) + + if summary_counts != dict(observed_counts): + raise ValueError( + "failure_summary['by_category'] must match failed train cases exactly; " + f"summary={summary_counts!r}, observed={dict(observed_counts)!r}" + ) + + +def _normalize_positive_count(category: object, value: object) -> int: + normalized = value if type(value) is int and value > 0 else None + + if normalized is None: + raise ValueError( + "failure_summary['by_category'] count must be a positive integer; " + f"category={category!r}, count={value!r}" + ) + return normalized diff --git a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py index 57f16e05..c8517743 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py +++ b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py @@ -1,5 +1,6 @@ from __future__ import annotations +import inspect import json from dataclasses import replace from pathlib import Path @@ -7,7 +8,10 @@ import pytest from trpc_agent_sdk.evaluation import EvalSet +from examples.optimization.eval_optimize_loop.eval_loop import fake_model as fake_model_module from examples.optimization.eval_optimize_loop.eval_loop.backends import FakeBackend +from examples.optimization.eval_optimize_loop.eval_loop.evaluator import ExampleEvaluator +from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel from examples.optimization.eval_optimize_loop.eval_loop.optimizer import FakeOptimizer from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult @@ -25,34 +29,14 @@ SAFE_PROMPT = f"{BASELINE_PROMPT}\n\n{SAFE_INSTRUCTION}." -def test_model_output_and_trace_ignore_all_evaluator_only_case_fields(): - input_text = "Return strict JSON with intent=refund and priority=high." - base = EvalCase( - case_id="oracle_case_a", - split="train", - input=input_text, - expectation={ - "type": "json", - "expected_values": {"secret": "first-oracle"}, - }, - tags=["first-tag"], - protected=False, - simulated_outputs={"safe": "FIRST SIMULATED ORACLE"}, - ) - variants = [ - base, - replace(base, case_id="oracle_case_b"), - replace( - base, - expectation={"type": "exact", "expected": "SECOND ORACLE"}, - ), - replace(base, split="validation"), - replace(base, protected=True), - replace(base, tags=["different", "oracle-tag"]), - replace(base, simulated_outputs={"safe": "SECOND SIMULATED ORACLE"}), - ] +def test_model_output_and_trace_are_stable_for_the_same_public_input(): + public_input = "Return strict JSON with intent=refund and priority=high." + same_inputs = [public_input, f"{public_input}", "".join([public_input])] - generated = [FakeModel(seed=91).generate("candidate", SAFE_PROMPT, case) for case in variants] + generated = [ + FakeModel(seed=91).generate("candidate", SAFE_PROMPT, user_input) + for user_input in same_inputs + ] assert {output for output, _, _ in generated} == {'{"intent": "refund", "priority": "high"}'} assert {json.dumps(trace, sort_keys=True) for _, trace, _ in generated} == { @@ -61,19 +45,72 @@ def test_model_output_and_trace_ignore_all_evaluator_only_case_fields(): sort_keys=True, ) } - serialized_traces = json.dumps([trace for _, trace, _ in generated]) - assert "ORACLE" not in serialized_traces - assert "oracle_case" not in serialized_traces + + +def test_example_evaluator_only_exposes_public_user_input_to_model(): + class SpyModel: + def __init__(self) -> None: + self.user_inputs: list[str] = [] + + def generate( + self, + prompt_id: str, + prompt: str, + user_input: str, + ) -> tuple[str, dict[str, object], float]: + assert isinstance(user_input, str) + self.user_inputs.append(user_input) + return "SECRET_EXPECTATION", {"prompt_id": prompt_id, "prompt": prompt}, 0.0 + + spy = SpyModel() + secret_case = EvalCase( + case_id="SECRET_CASE_ID", + split="validation", + input="PUBLIC USER INPUT", + expectation={"type": "exact", "expected": "SECRET_EXPECTATION"}, + tags=["SECRET_TAG"], + protected=True, + simulated_outputs={"safe": "SECRET_SIMULATED_OUTPUT"}, + ) + + result = ExampleEvaluator(spy, FakeJudge()).evaluate( + prompt_id="candidate", + prompt="PUBLIC SYSTEM PROMPT", + cases=[secret_case], + split="validation", + ) + + assert result.passed is True + assert spy.user_inputs == ["PUBLIC USER INPUT"] + assert "SECRET" not in json.dumps(spy.user_inputs) + + +def test_fake_model_source_has_no_evaluator_oracle_capability(): + model_source = inspect.getsource(fake_model_module) + evaluator_source = inspect.getsource(ExampleEvaluator.evaluate) + + assert "EvalCase" not in model_source + assert ".expectation" not in model_source + assert ".tags" not in model_source + assert ".protected" not in model_source + assert ".simulated_outputs" not in model_source + assert "self.model.generate(prompt_id, prompt, case.input)" in evaluator_source + + +@pytest.mark.parametrize("user_input", [None, b"bytes", object()]) +def test_fake_model_rejects_non_string_user_input(user_input: object): + with pytest.raises(TypeError, match="^user_input must be a string$"): + FakeModel(seed=91).generate("candidate", SAFE_PROMPT, user_input) def test_model_is_deterministic_and_prompt_id_or_seed_cannot_change_output(): - case = _case("Return strict JSON with status=READY and next_step=ship.") + user_input = "Return strict JSON with status=READY and next_step=ship." model = FakeModel(seed=91) - first = model.generate("first-id", SAFE_PROMPT, case) - repeated = model.generate("first-id", SAFE_PROMPT, case) - different_prompt_id = model.generate("second-id", SAFE_PROMPT, case) - different_seed = FakeModel(seed=999).generate("first-id", SAFE_PROMPT, case) + first = model.generate("first-id", SAFE_PROMPT, user_input) + repeated = model.generate("first-id", SAFE_PROMPT, user_input) + different_prompt_id = model.generate("second-id", SAFE_PROMPT, user_input) + different_seed = FakeModel(seed=999).generate("first-id", SAFE_PROMPT, user_input) assert first == repeated assert first[0] == different_prompt_id[0] == different_seed[0] @@ -95,9 +132,9 @@ def test_strict_json_rendering_uses_assignments_from_user_input( prompt: str, expected: str, ): - case = _case("Return StRiCt JsOn with intent=refund and priority=high.") + user_input = "Return StRiCt JsOn with intent=refund and priority=high." - output, _, _ = FakeModel(seed=91).generate("arbitrary-id", prompt, case) + output, _, _ = FakeModel(seed=91).generate("arbitrary-id", prompt, user_input) assert output == expected @@ -111,9 +148,9 @@ def test_strict_json_rendering_uses_assignments_from_user_input( ], ) def test_return_only_rendering_is_prompt_mode_sensitive(prompt: str, expected: str): - case = _case("ReTuRn OnLy YES; do not use JSON.") + user_input = "ReTuRn OnLy YES; do not use JSON." - output, _, _ = FakeModel(seed=91).generate("arbitrary-id", prompt, case) + output, _, _ = FakeModel(seed=91).generate("arbitrary-id", prompt, user_input) assert output == expected @@ -133,11 +170,9 @@ def test_return_only_rendering_is_prompt_mode_sensitive(prompt: str, expected: s ], ) def test_natural_and_unknown_requests_are_deterministic(input_text: str, natural: str): - case = _case(input_text) - - baseline, _, _ = FakeModel(seed=91).generate("baseline", BASELINE_PROMPT, case) - safe, _, _ = FakeModel(seed=91).generate("safe", SAFE_PROMPT, case) - overfit, _, _ = FakeModel(seed=91).generate("overfit", OVERFIT_PROMPT, case) + baseline, _, _ = FakeModel(seed=91).generate("baseline", BASELINE_PROMPT, input_text) + safe, _, _ = FakeModel(seed=91).generate("safe", SAFE_PROMPT, input_text) + overfit, _, _ = FakeModel(seed=91).generate("overfit", OVERFIT_PROMPT, input_text) assert baseline == safe == natural assert json.loads(overfit) == {"answer": natural} @@ -159,15 +194,23 @@ def test_optimizer_returns_no_candidates_without_observed_target_failures(): failure_category="llm_rubric_not_met", ) ) + unclassified_failure = _eval_result( + _case_result( + "unclassified_failure", + passed=False, + failure_category=None, + ) + ) assert ( optimizer.propose( BASELINE_PROMPT, all_pass, - {"by_category": {"format_violation": 1}}, + {}, ) == [] ) + assert optimizer.propose(BASELINE_PROMPT, unclassified_failure, {}) == [] assert ( optimizer.propose( BASELINE_PROMPT, @@ -178,22 +221,149 @@ def test_optimizer_returns_no_candidates_without_observed_target_failures(): ) +def test_optimizer_rejects_validation_result_as_training_evidence(): + baseline_validation = replace(_eval_result(), split="validation") + + with pytest.raises(ValueError, match="baseline_train.split.*train.*validation"): + FakeOptimizer().propose( + BASELINE_PROMPT, + baseline_validation, + {}, + ) + + +def test_optimizer_rejects_validation_case_mixed_into_training_evidence(): + mixed_result = _eval_result( + _case_result( + "train_case", + passed=False, + failure_category="format_violation", + ), + replace( + _case_result( + "validation_case", + passed=False, + failure_category="format_violation", + ), + split="validation", + ), + ) + + with pytest.raises(ValueError, match="case.*validation_case.*split.*validation"): + FakeOptimizer().propose( + BASELINE_PROMPT, + mixed_result, + {}, + ) + + +@pytest.mark.parametrize("failure_summary", [None, [], "not-a-summary"]) +def test_optimizer_rejects_non_mapping_failure_summary( + failure_summary: object, +): + observed_failure = _eval_result( + _case_result( + "observed_failure", + passed=False, + failure_category="format_violation", + ) + ) + + with pytest.raises(TypeError, match="^failure_summary must be a dict$"): + FakeOptimizer().propose( + BASELINE_PROMPT, + observed_failure, + failure_summary, + ) + + +def test_optimizer_rejects_summary_category_not_observed_in_failed_cases(): + unclassified_failure = _eval_result( + _case_result( + "unclassified_failure", + passed=False, + failure_category=None, + ) + ) + + with pytest.raises(ValueError, match="by_category.*failed train cases"): + FakeOptimizer().propose( + BASELINE_PROMPT, + unclassified_failure, + {"by_category": {"format_violation": 1}}, + ) + + @pytest.mark.parametrize( - ("case_failure_category", "failure_summary"), + "by_category", [ - ("format_violation", {}), - (None, {"by_category": {"final_response_mismatch": 2}}), + {"final_response_mismatch": 1}, + {"format_violation": 2}, + {}, ], ) -def test_optimizer_proposes_natural_language_candidates_from_observed_failures( - case_failure_category: str | None, +def test_optimizer_rejects_summary_category_or_count_mismatch( + by_category: dict[str, object], +): + observed_failure = _eval_result( + _case_result( + "observed_failure", + passed=False, + failure_category="format_violation", + ) + ) + + with pytest.raises(ValueError, match="by_category.*failed train cases"): + FakeOptimizer().propose( + BASELINE_PROMPT, + observed_failure, + {"by_category": by_category}, + ) + + +@pytest.mark.parametrize("by_category", [None, [], "format_violation"]) +def test_optimizer_rejects_non_mapping_by_category(by_category: object): + with pytest.raises(ValueError, match="failure_summary.*by_category.*dict"): + FakeOptimizer().propose( + BASELINE_PROMPT, + _eval_result(), + {"by_category": by_category}, + ) + + +@pytest.mark.parametrize("count", [True, 0, -1, 1.0, 1.5, "1", None]) +def test_optimizer_rejects_non_positive_integer_summary_counts(count: object): + observed_failure = _eval_result( + _case_result( + "observed_failure", + passed=False, + failure_category="format_violation", + ) + ) + + with pytest.raises(ValueError, match="count.*positive integer"): + FakeOptimizer().propose( + BASELINE_PROMPT, + observed_failure, + {"by_category": {"format_violation": count}}, + ) + + +@pytest.mark.parametrize( + "failure_summary", + [ + {}, + {"by_category": {"format_violation": 1}}, + ], +) +def test_optimizer_proposes_two_candidates_for_observed_train_format_failure( failure_summary: dict[str, object], ): baseline_train = _eval_result( _case_result( "random_training_case", passed=False, - failure_category=case_failure_category, + failure_category="format_violation", ) ) candidates = FakeOptimizer().propose( @@ -338,15 +508,6 @@ async def test_fake_backend_scores_baseline_overfit_and_safe_on_official_data( ) -def _case(input_text: str) -> EvalCase: - return EvalCase( - case_id="arbitrary-case-id", - split="train", - input=input_text, - expectation={"type": "exact", "expected": "EVALUATOR ORACLE"}, - ) - - def _case_result( case_id: str, *, From 4563b0c3e15711a8d73834795fd20236389701df Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 01:06:40 +0800 Subject: [PATCH 24/34] refactor(examples): unify evaluation optimization pipeline --- .../eval_optimize_loop/eval_loop/gate.py | 127 ++- .../eval_optimize_loop/eval_loop/pipeline.py | 622 +++++++++++++ .../eval_optimize_loop/eval_loop/report.py | 176 +++- .../eval_optimize_loop/run_pipeline.py | 877 ++++-------------- .../eval_optimize_loop/tests/test_gate.py | 210 +++++ .../tests/test_pipeline_fake_mode.py | 48 +- .../tests/test_pipeline_orchestration.py | 697 ++++++++++++++ .../tests/test_sdk_backend.py | 213 +++-- 8 files changed, 2169 insertions(+), 801 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py diff --git a/examples/optimization/eval_optimize_loop/eval_loop/gate.py b/examples/optimization/eval_optimize_loop/eval_loop/gate.py index ae9abb72..5f12539b 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/gate.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/gate.py @@ -5,6 +5,7 @@ from typing import Any from .schemas import CaseDelta +from .schemas import CostSummary from .schemas import EvalResult from .schemas import GateDecision @@ -35,6 +36,7 @@ def decide( candidate_train: EvalResult, candidate_validation: EvalResult, deltas: list[CaseDelta], + cost_summary: CostSummary, cumulative_cost: float = 0.0, ) -> GateDecision: train_delta = round(candidate_train.score - baseline_train.score, 6) @@ -42,10 +44,24 @@ def decide( candidate_cost = round(candidate_train.cost + candidate_validation.cost, 6) reasons: list[str] = [] + train_comparable = _append_comparability_reasons( + reasons, + split="train", + baseline=baseline_train, + candidate=candidate_train, + ) + validation_comparable = _append_comparability_reasons( + reasons, + split="validation", + baseline=baseline_validation, + candidate=candidate_validation, + ) + overfit_detected = train_delta > 0 and val_delta <= 0 if overfit_detected: reasons.append( - "reject: overfit detected because train score improved but validation score regressed or did not improve " + "reject: overfit detected because train score improved but " + "validation score regressed or did not improve " f"({train_delta:+.3f} train, {val_delta:+.3f} validation)" ) @@ -56,20 +72,32 @@ def decide( f"{val_delta:+.3f} is below required {min_val_improvement:+.3f}" ) - baseline_validation_by_id = baseline_validation.by_case_id() - candidate_validation_by_id = candidate_validation.by_case_id() - validation_new_failures = [ - case_id - for case_id, candidate_case in sorted(candidate_validation_by_id.items()) - if not candidate_case.passed and baseline_validation_by_id.get(case_id) - and baseline_validation_by_id[case_id].passed - ] - new_hard_failures = [ - case_id - for case_id, candidate_case in sorted(candidate_validation_by_id.items()) - if candidate_case.hard_failed and baseline_validation_by_id.get(case_id) - and baseline_validation_by_id[case_id].passed - ] + validation_new_failures: list[str] = [] + if validation_comparable: + baseline_validation_by_id = baseline_validation.by_case_id() + candidate_validation_by_id = candidate_validation.by_case_id() + validation_new_failures = [ + case_id + for case_id, candidate_case in sorted(candidate_validation_by_id.items()) + if not candidate_case.passed and baseline_validation_by_id[case_id].passed + ] + if validation_new_failures: + reasons.append(f"reject: new validation failures appeared: {validation_new_failures}") + + new_hard_failures: list[str] = [] + for baseline_result, candidate_result, comparable in ( + (baseline_train, candidate_train, train_comparable), + (baseline_validation, candidate_validation, validation_comparable), + ): + if not comparable: + continue + baseline_by_id = baseline_result.by_case_id() + new_hard_failures.extend( + case_id + for case_id, candidate_case in sorted(candidate_result.by_case_id().items()) + if candidate_case.hard_failed and not baseline_by_id[case_id].hard_failed + ) + new_hard_failures = sorted(set(new_hard_failures)) if new_hard_failures and not bool(self.config["allow_new_hard_fail"]): reasons.append(f"reject: new hard failures appeared: {new_hard_failures}") @@ -91,10 +119,16 @@ def decide( if excessive_drops: reasons.append(f"reject: per-case validation score drops exceed {max_drop:.3f}: {excessive_drops}") - max_total_cost = float(self.config["max_total_cost"]) total_run_cost = round(cumulative_cost + candidate_cost, 6) - if total_run_cost > max_total_cost: - reasons.append(f"reject: total run cost {total_run_cost:.3f} exceeds budget {max_total_cost:.3f}") + configured_max_total_cost = self.config["max_total_cost"] + if configured_max_total_cost is not None: + max_total_cost = float(configured_max_total_cost) + if not cost_summary.complete: + reasons.append("reject: cost_unavailable for configured max_total_cost") + elif total_run_cost > max_total_cost: + reasons.append( + f"reject: total run cost {total_run_cost:.3f} exceeds budget {max_total_cost:.3f}" + ) accepted = not any(reason.startswith("reject:") for reason in reasons) if accepted: @@ -118,4 +152,61 @@ def decide( cumulative_cost=round(cumulative_cost, 6), total_run_cost=total_run_cost, cost=candidate_cost, + gate_status="applied", + gate_not_applied_reason=None, + not_applied_checks=[], + ) + + +def _append_comparability_reasons( + reasons: list[str], + *, + split: str, + baseline: EvalResult, + candidate: EvalResult, +) -> bool: + """Reject malformed result pairs before any dict conversion can hide evidence.""" + + comparable = True + baseline_ids = [case.case_id for case in baseline.cases] + candidate_ids = [case.case_id for case in candidate.cases] + if not baseline_ids: + reasons.append(f"reject: baseline {split} has empty case results") + comparable = False + if not candidate_ids: + reasons.append(f"reject: candidate {split} has empty case results") + comparable = False + + baseline_duplicates = _duplicates(baseline_ids) + candidate_duplicates = _duplicates(candidate_ids) + if baseline_duplicates: + reasons.append( + f"reject: baseline {split} has duplicate case IDs: {baseline_duplicates}" + ) + comparable = False + if candidate_duplicates: + reasons.append( + f"reject: candidate {split} has duplicate case IDs: {candidate_duplicates}" + ) + comparable = False + + baseline_set = set(baseline_ids) + candidate_set = set(candidate_ids) + if baseline_set != candidate_set: + reasons.append( + f"reject: {split} case ID set mismatch; " + f"missing={sorted(baseline_set - candidate_set)}, " + f"extra={sorted(candidate_set - baseline_set)}" ) + comparable = False + return comparable + + +def _duplicates(case_ids: list[str]) -> list[str]: + seen: set[str] = set() + duplicates: set[str] = set() + for case_id in case_ids: + if case_id in seen: + duplicates.add(case_id) + seen.add(case_id) + return sorted(duplicates) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py new file mode 100644 index 00000000..2b7878f4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py @@ -0,0 +1,622 @@ +"""Backend-neutral asynchronous evaluation and optimization orchestration.""" + +from __future__ import annotations + +import hashlib +import re +import shlex +import time +from dataclasses import dataclass +from dataclasses import replace +from pathlib import Path +from typing import Any + +from .attribution import summarize_failures +from .backends import EvaluationBackend +from .backends import OptimizationBackend +from .gate import AcceptanceGate +from .loader import read_json +from .loader import sha256_file +from .loader import stable_config_hash +from .report import build_report +from .report import compute_case_deltas +from .report import finalize_run_artifacts +from .report import prepare_run_artifacts +from .schemas import CandidatePrompt +from .schemas import CostSummary +from .schemas import EvalResult +from .schemas import OptimizationReport +from .schemas import WritebackResult +from .schemas import to_jsonable +from .writeback import commit_prompt_bundle +from .writeback import snapshot_prompt_files + + +_ARTIFACT_COMPONENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +@dataclass(frozen=True) +class PipelineRequest: + """All inputs required by the shared asynchronous pipeline.""" + + train_path: str | Path + validation_path: str | Path + optimizer_config_path: str | Path + output_dir: str | Path + target_prompt_paths: dict[str, str | Path] + gate_config: dict[str, Any] + trace: bool + update_source: bool + mode: str + run_id: str + sdk_call_agent: str | None = None + gate_config_path: str | Path | None = None + + +async def execute_pipeline( + request: PipelineRequest, + *, + evaluator: EvaluationBackend, + optimizer: OptimizationBackend, +) -> OptimizationReport: + """Evaluate, optimize, gate, audit, and optionally commit one prompt bundle.""" + + started = time.perf_counter() + run_id = _safe_artifact_component(request.run_id, context="run_id") + if not request.target_prompt_paths: + raise ValueError("target_prompt_paths must not be empty") + + snapshot = snapshot_prompt_files(request.target_prompt_paths) + baseline_prompts = _decode_snapshot(snapshot) + config_snapshot = read_json(request.optimizer_config_path) + effective_gate_config = _gate_config_with_dataset_protection( + request.gate_config, + validation_path=request.validation_path, + ) + run_root = Path(request.output_dir) / "runs" / run_id + + baseline_train = await evaluator.evaluate( + prompt_id="baseline", + prompts=baseline_prompts, + dataset_path=request.train_path, + split="train", + trace=request.trace, + artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "train"), + ) + _validate_result_identity( + baseline_train, + expected_prompt_id="baseline", + expected_split="train", + ) + baseline_validation = await evaluator.evaluate( + prompt_id="baseline", + prompts=baseline_prompts, + dataset_path=request.validation_path, + split="validation", + trace=request.trace, + artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "validation"), + ) + _validate_result_identity( + baseline_validation, + expected_prompt_id="baseline", + expected_split="validation", + ) + + failure_summary = summarize_failures([baseline_train]) + optimizer_artifact_dir = _artifact_dir(run_root, "optimizer") + optimization = await optimizer.optimize_candidates( + baseline_prompts=baseline_prompts, + baseline_train=baseline_train, + failure_summary=failure_summary, + train_path=request.train_path, + validation_path=request.validation_path, + config_path=request.optimizer_config_path, + artifact_dir=optimizer_artifact_dir, + ) + candidate_bundles = _validated_candidate_bundles( + optimization.candidates, + expected_fields=set(snapshot.files), + ) + + gate = AcceptanceGate(effective_gate_config) + baseline_evaluator_cost = round(baseline_train.cost + baseline_validation.cost, 6) + explicit_evaluator_cost = baseline_evaluator_cost + cumulative_cost = round(baseline_evaluator_cost + optimization.cost.total, 6) + candidate_records: list[dict[str, Any]] = [] + all_deltas = [] + gate_decisions = [] + all_results_comparable = True + + for index, candidate in enumerate(optimization.candidates, start=1): + bundle = candidate_bundles[candidate.candidate_id] + path_label = f"{index:03d}_{_path_label(candidate.candidate_id)}" + train_result = await evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=bundle, + dataset_path=request.train_path, + split="train", + trace=request.trace, + artifact_dir=_artifact_dir(run_root, "evaluations", path_label, "train"), + ) + _validate_result_identity( + train_result, + expected_prompt_id=candidate.candidate_id, + expected_split="train", + ) + validation_result = await evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=bundle, + dataset_path=request.validation_path, + split="validation", + trace=request.trace, + artifact_dir=_artifact_dir(run_root, "evaluations", path_label, "validation"), + ) + _validate_result_identity( + validation_result, + expected_prompt_id=candidate.candidate_id, + expected_split="validation", + ) + comparable = _result_pair_is_comparable(baseline_train, train_result) and ( + _result_pair_is_comparable(baseline_validation, validation_result) + ) + all_results_comparable = all_results_comparable and comparable + deltas = ( + compute_case_deltas( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + ) + if comparable + else [] + ) + decision = gate.decide( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + deltas=deltas, + cost_summary=optimization.cost, + cumulative_cost=cumulative_cost, + ) + candidate_records.append({ + "candidate": candidate, + "train_result": train_result, + "validation_result": validation_result, + }) + all_deltas.extend(deltas) + gate_decisions.append(decision) + candidate_eval_cost = round(train_result.cost + validation_result.cost, 6) + explicit_evaluator_cost = round(explicit_evaluator_cost + candidate_eval_cost, 6) + cumulative_cost = decision.total_run_cost + + selected_candidate = _select_candidate(candidate_records, gate_decisions) + cost_summary = CostSummary( + optimizer=optimization.cost.optimizer, + evaluator=round(optimization.cost.evaluator + explicit_evaluator_cost, 6), + agent=optimization.cost.agent, + total=cumulative_cost, + complete=optimization.cost.complete, + ) + input_hashes: dict[str, Any] = { + "train": sha256_file(request.train_path), + "validation": sha256_file(request.validation_path), + "optimizer": sha256_file(request.optimizer_config_path), + "target_prompts": snapshot.hashes(), + } + if request.gate_config_path is not None: + input_hashes["gate_config"] = sha256_file(request.gate_config_path) + before_hashes = snapshot.hashes() + writeback_journal = _initial_writeback_journal( + selected_candidate=selected_candidate, + update_source=request.update_source, + before_hashes=before_hashes, + ) + audit = _build_audit( + request=request, + started=started, + snapshot_hashes=snapshot.hashes(), + input_hashes=input_hashes, + config_snapshot=config_snapshot, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + candidate_bundles=candidate_bundles, + optimization_raw_summary=optimization.raw_summary, + optimization_cost=optimization.cost, + cost_summary=cost_summary, + all_results_comparable=all_results_comparable, + effective_gate_config=effective_gate_config, + writeback_journal=writeback_journal, + ) + run = _build_run( + request, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + ) + if selected_candidate is None: + provisional_writeback = WritebackResult( + status="rejected", + before_hashes=before_hashes, + ) + else: + provisional_writeback = WritebackResult( + status="not_requested", + before_hashes=before_hashes, + error=( + "pending source writeback; pre-write audit must be prepared before commit" + if request.update_source + else None + ), + ) + report = build_report( + run=run, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + per_case_deltas=all_deltas, + gate_decisions=gate_decisions, + selected_candidate=selected_candidate, + audit=audit, + rounds=optimization.rounds, + cost_summary=cost_summary, + writeback=provisional_writeback, + ) + + artifact_paths = prepare_run_artifacts(report, request.output_dir) + if selected_candidate is None: + writeback = provisional_writeback + elif not request.update_source: + writeback = provisional_writeback + else: + writeback = commit_prompt_bundle( + snapshot, + candidate_bundles[selected_candidate], + ) + final_audit = dict(report.audit) + final_journal = dict(writeback_journal) + final_journal.update({ + "state": writeback.status, + "after_hashes": dict(writeback.after_hashes), + "error": writeback.error, + }) + final_audit["writeback_journal"] = final_journal + final_report = replace(report, audit=final_audit, writeback=writeback) + finalize_run_artifacts(final_report, artifact_paths) + return final_report + + +def _decode_snapshot(snapshot: Any) -> dict[str, str]: + prompts: dict[str, str] = {} + for name, prompt_file in snapshot.files.items(): + try: + prompts[name] = prompt_file.content.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise ValueError( + f"target prompt {name!r} at {prompt_file.path} is not valid UTF-8" + ) from exc + return prompts + + +def _validated_candidate_bundles( + candidates: list[CandidatePrompt], + *, + expected_fields: set[str], +) -> dict[str, dict[str, str]]: + bundles: dict[str, dict[str, str]] = {} + for candidate in candidates: + candidate_id = candidate.candidate_id + if not isinstance(candidate_id, str) or not candidate_id: + raise ValueError("candidate_id must be a non-empty string") + if candidate_id == "baseline": + raise ValueError("candidate_id 'baseline' is reserved") + if candidate_id in bundles: + raise ValueError(f"duplicate candidate_id: {candidate_id}") + bundle = candidate.bundle() + actual_fields = set(bundle) + if actual_fields != expected_fields: + raise ValueError( + f"candidate {candidate_id!r} bundle fields must exactly match target prompt fields; " + f"missing={sorted(expected_fields - actual_fields)}, " + f"extra={sorted(actual_fields - expected_fields)}" + ) + for field_name, prompt_text in bundle.items(): + if not isinstance(prompt_text, str) or not prompt_text: + raise ValueError( + f"candidate {candidate_id!r} bundle field {field_name!r} " + "must be a non-empty string" + ) + bundles[candidate_id] = dict(bundle) + return bundles + + +def _result_pair_is_comparable(baseline: EvalResult, candidate: EvalResult) -> bool: + baseline_ids = [case.case_id for case in baseline.cases] + candidate_ids = [case.case_id for case in candidate.cases] + return bool(baseline_ids) and bool(candidate_ids) and ( + len(baseline_ids) == len(set(baseline_ids)) + and len(candidate_ids) == len(set(candidate_ids)) + and set(baseline_ids) == set(candidate_ids) + ) + + +def _validate_result_identity( + result: EvalResult, + *, + expected_prompt_id: str, + expected_split: str, +) -> None: + if result.prompt_id != expected_prompt_id: + raise ValueError( + f"evaluation result prompt_id mismatch: expected {expected_prompt_id!r}, " + f"got {result.prompt_id!r}" + ) + if result.split != expected_split: + raise ValueError( + f"EvalResult split mismatch for {expected_prompt_id!r}: " + f"expected {expected_split!r}, got {result.split!r}" + ) + for case in result.cases: + if case.split != expected_split: + raise ValueError( + f"CaseResult {case.case_id!r} split mismatch for {expected_prompt_id!r}: " + f"expected {expected_split!r}, got {case.split!r}" + ) + + +def _gate_config_with_dataset_protection( + gate_config: dict[str, Any], + *, + validation_path: str | Path, +) -> dict[str, Any]: + effective = dict(gate_config) + configured_ids = effective.get("protected_case_ids", []) + if not isinstance(configured_ids, list) or not all( + isinstance(case_id, str) for case_id in configured_ids + ): + raise ValueError("gate protected_case_ids must be a list of strings") + protected_ids = set(configured_ids) + payload = read_json(validation_path) + standard_cases = payload.get("eval_cases") + legacy_cases = payload.get("cases") + if isinstance(standard_cases, list): + for case in standard_cases: + if not isinstance(case, dict): + continue + session_input = case.get("session_input") + state = session_input.get("state") if isinstance(session_input, dict) else None + protected = state.get("eval_optimize_protected", False) if isinstance(state, dict) else False + if protected is True: + case_id = case.get("eval_id") + if isinstance(case_id, str) and case_id: + protected_ids.add(case_id) + elif isinstance(legacy_cases, list): + for case in legacy_cases: + if not isinstance(case, dict) or case.get("protected") is not True: + continue + case_id = case.get("case_id") or case.get("id") + if isinstance(case_id, str) and case_id: + protected_ids.add(case_id) + effective["protected_case_ids"] = sorted(protected_ids) + return effective + + +def _select_candidate( + candidate_records: list[dict[str, Any]], + gate_decisions: list[Any], +) -> str | None: + decisions_by_id = {decision.candidate_id: decision for decision in gate_decisions} + accepted: list[tuple[int, dict[str, Any]]] = [] + for index, record in enumerate(candidate_records): + candidate = record["candidate"] + if decisions_by_id[candidate.candidate_id].accepted: + accepted.append((index, record)) + if not accepted: + return None + _, selected = max( + accepted, + key=lambda item: ( + item[1]["validation_result"].score, + item[1]["train_result"].score, + -item[0], + ), + ) + return selected["candidate"].candidate_id + + +def _build_run( + request: PipelineRequest, + *, + baseline_train: EvalResult, + baseline_validation: EvalResult, +) -> dict[str, Any]: + target_paths = {name: str(path) for name, path in request.target_prompt_paths.items()} + return { + "run_id": request.run_id, + "mode": request.mode, + "trace": request.trace, + "trace_enabled": request.trace, + "update_source": request.update_source, + "train_cases": len(baseline_train.cases), + "validation_cases": len(baseline_validation.cases), + "target_prompt_paths": target_paths, + "prompt_source": target_paths.get("system_prompt"), + "paths": { + "train": str(request.train_path), + "validation": str(request.validation_path), + "optimizer": str(request.optimizer_config_path), + "prompts": target_paths, + }, + "reproducibility_command": ( + _reproducibility_command(request) + ), + } + + +def _build_audit( + *, + request: PipelineRequest, + started: float, + snapshot_hashes: dict[str, str], + input_hashes: dict[str, Any], + config_snapshot: dict[str, Any], + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_records: list[dict[str, Any]], + candidate_bundles: dict[str, dict[str, str]], + optimization_raw_summary: dict[str, Any], + optimization_cost: CostSummary, + cost_summary: CostSummary, + all_results_comparable: bool, + effective_gate_config: dict[str, Any], + writeback_journal: dict[str, Any], +) -> dict[str, Any]: + candidate_costs = { + record["candidate"].candidate_id: round( + record["train_result"].cost + record["validation_result"].cost, + 6, + ) + for record in candidate_records + } + candidate_prompt_hashes = { + candidate_id: { + name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() + for name, prompt in bundle.items() + } + for candidate_id, bundle in candidate_bundles.items() + } + duration = max(time.perf_counter() - started, 1e-9) + target_paths = {name: str(path) for name, path in request.target_prompt_paths.items()} + return { + "seed": _config_seed(config_snapshot), + "duration_seconds": duration, + "config_hash": stable_config_hash(config_snapshot), + "config_snapshot": config_snapshot, + "gate_config_hash": stable_config_hash(effective_gate_config), + "gate_config_snapshot": effective_gate_config, + "writeback_journal": writeback_journal, + "input_hashes": input_hashes, + "input_paths": { + "train": str(request.train_path), + "validation": str(request.validation_path), + "optimizer": str(request.optimizer_config_path), + "prompts": target_paths, + **({"prompt": target_paths["system_prompt"]} if "system_prompt" in target_paths else {}), + }, + "prompt_hash": snapshot_hashes.get("system_prompt"), + "prompt_hashes": snapshot_hashes, + "candidate_prompt_hashes": candidate_prompt_hashes, + "candidate_prompts": candidate_bundles, + "prompt_diffs": { + record["candidate"].candidate_id: record["candidate"].prompt_diff + for record in candidate_records + }, + "total_run_cost": cost_summary.total, + "cost": { + "baseline": round(baseline_train.cost + baseline_validation.cost, 6), + "candidates": candidate_costs, + "optimization": to_jsonable(optimization_cost), + "evaluator": cost_summary.evaluator, + "total": cost_summary.total, + "complete": cost_summary.complete, + }, + "sdk_result_summary": to_jsonable(optimization_raw_summary), + "sdk_result_availability": { + "aggregate_validation_result": bool( + request.mode == "sdk" and optimization_raw_summary + ), + "full_train_eval_result": True, + "full_per_case_validation_delta": all_results_comparable, + }, + "reproducibility_command": ( + _reproducibility_command(request) + ), + } + + +def _artifact_dir(root: Path, *parts: str) -> Path: + path = root.joinpath(*parts) + path.mkdir(parents=True, exist_ok=True) + return path + + +def _safe_artifact_component(value: str, *, context: str) -> str: + if value in {"", ".", ".."} or not _ARTIFACT_COMPONENT_RE.fullmatch(value): + raise ValueError(f"unsafe {context}: {value!r}") + return value + + +def _path_label(value: str) -> str: + label = re.sub(r"[^A-Za-z0-9_.-]", "_", value) + if label in {"", ".", ".."}: + return "candidate" + return label + + +def _config_seed(config_snapshot: dict[str, Any]) -> Any: + if "seed" in config_snapshot: + return config_snapshot["seed"] + optimize = config_snapshot.get("optimize") + if isinstance(optimize, dict): + algorithm = optimize.get("algorithm") + if isinstance(algorithm, dict) and "seed" in algorithm: + return algorithm["seed"] + return None + + +def _reproducibility_command(request: PipelineRequest) -> str: + command = [ + "python", + "examples/optimization/eval_optimize_loop/run_pipeline.py", + "--mode", + request.mode, + "--train", + str(request.train_path), + "--val", + str(request.validation_path), + "--optimizer-config", + str(request.optimizer_config_path), + "--output-dir", + str(request.output_dir), + "--run-id", + request.run_id, + ] + target_paths = list(request.target_prompt_paths.items()) + if set(request.target_prompt_paths) == {"system_prompt"}: + command.extend(["--prompt", str(request.target_prompt_paths["system_prompt"])]) + else: + for name, path in target_paths: + command.extend(["--target-prompt", f"{name}={path}"]) + if request.trace: + command.append("--trace") + if request.mode == "sdk" and request.sdk_call_agent: + command.extend(["--sdk-call-agent", request.sdk_call_agent]) + if request.gate_config_path is not None: + command.extend(["--gate-config", str(request.gate_config_path)]) + if request.update_source: + command.append("--update-source") + return shlex.join(command) + + +def _initial_writeback_journal( + *, + selected_candidate: str | None, + update_source: bool, + before_hashes: dict[str, str], +) -> dict[str, Any]: + if selected_candidate is None: + state = "rejected" + elif not update_source: + state = "not_requested" + else: + state = "pending" + return { + "state": state, + "requested": update_source, + "selected_candidate": selected_candidate, + "before_hashes": dict(before_hashes), + "after_hashes": {}, + "error": None, + } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index de1a1b35..cd554673 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -5,15 +5,19 @@ import json import re import shutil +from dataclasses import dataclass from pathlib import Path from typing import Any from .attribution import summarize_failures from .schemas import CandidatePrompt from .schemas import CaseDelta +from .schemas import CostSummary from .schemas import EvalResult from .schemas import GateDecision from .schemas import OptimizationReport +from .schemas import OptimizationRound +from .schemas import WritebackResult from .schemas import to_jsonable @@ -29,6 +33,23 @@ ARTIFACT_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +@dataclass(frozen=True) +class RunArtifactPaths: + """Locations written by the audit-first report lifecycle.""" + + output_dir: Path + run_dir: Path + prewrite_json: Path + prewrite_markdown: Path + audit_json: Path + final_json: Path + final_markdown: Path + writeback_json: Path + writeback_journal: Path + top_level_json: Path + top_level_markdown: Path + + def compute_case_deltas( *, candidate_id: str, @@ -75,6 +96,9 @@ def build_report( gate_decisions: list[GateDecision], selected_candidate: str | None, audit: dict[str, Any], + rounds: list[OptimizationRound] | None = None, + cost_summary: CostSummary | None = None, + writeback: WritebackResult | None = None, ) -> OptimizationReport: all_results: list[EvalResult] = [baseline_train, baseline_validation] for record in candidate_records: @@ -93,18 +117,121 @@ def build_report( gate_decisions=gate_decisions, selected_candidate=selected_candidate, audit=audit, + rounds=list(rounds or []), + cost_summary=cost_summary or CostSummary(), + writeback=writeback or WritebackResult(status="not_requested"), ) def write_reports(report: OptimizationReport, output_dir: str | Path) -> tuple[Path, Path]: + """Compatibility helper for callers that do not perform source writeback.""" + + paths = prepare_run_artifacts(report, output_dir) + finalize_run_artifacts(report, paths) + return paths.top_level_json, paths.top_level_markdown + + +def prepare_run_artifacts( + report: OptimizationReport, + output_dir: str | Path, +) -> RunArtifactPaths: + """Persist a complete pre-write audit before any source prompt commit.""" + output_path = Path(output_dir) - output_path.mkdir(parents=True, exist_ok=True) - json_path = output_path / "optimization_report.json" - md_path = output_path / "optimization_report.md" - json_path.write_text(report_to_json(report), encoding="utf-8") - md_path.write_text(render_markdown(report), encoding="utf-8") + run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) + run_dir = output_path / "runs" / run_id + run_dir.mkdir(parents=True, exist_ok=True) + paths = RunArtifactPaths( + output_dir=output_path, + run_dir=run_dir, + prewrite_json=run_dir / "pre_write_report.json", + prewrite_markdown=run_dir / "pre_write_report.md", + audit_json=run_dir / "audit.json", + final_json=run_dir / "optimization_report.json", + final_markdown=run_dir / "optimization_report.md", + writeback_json=run_dir / "writeback.json", + writeback_journal=run_dir / "writeback_journal.json", + top_level_json=output_path / "optimization_report.json", + top_level_markdown=output_path / "optimization_report.md", + ) + paths.prewrite_json.write_text(report_to_json(report), encoding="utf-8") + paths.prewrite_markdown.write_text(render_markdown(report), encoding="utf-8") + paths.audit_json.write_text( + json.dumps( + to_jsonable(report.audit), + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + journal = dict(report.audit.get("writeback_journal", {})) + if journal.get("state") == "pending": + journal["state"] = "prepared" + paths.writeback_journal.write_text( + json.dumps( + to_jsonable(journal), + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) write_audit_artifacts(report, output_path) - return json_path, md_path + return paths + + +def finalize_run_artifacts(report: OptimizationReport, paths: RunArtifactPaths) -> None: + """Persist final writeback state and refresh top-level convenience copies.""" + + expected_run_dir = paths.output_dir / "runs" / _safe_artifact_name( + str(report.run.get("run_id") or "run") + ) + if paths.run_dir != expected_run_dir: + raise ValueError("run artifact paths do not match report run_id") + paths.writeback_json.write_text( + json.dumps( + to_jsonable(report.writeback), + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + paths.writeback_journal.write_text( + json.dumps( + to_jsonable(report.audit.get("writeback_journal", {})), + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + paths.audit_json.write_text( + json.dumps( + to_jsonable(report.audit), + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + paths.final_json.write_text(report_to_json(report), encoding="utf-8") + paths.final_markdown.write_text(render_markdown(report), encoding="utf-8") + paths.output_dir.mkdir(parents=True, exist_ok=True) + paths.top_level_json.write_text(report_to_json(report), encoding="utf-8") + paths.top_level_markdown.write_text(render_markdown(report), encoding="utf-8") + write_audit_artifacts(report, paths.output_dir) def report_to_json(report: OptimizationReport) -> str: @@ -130,12 +257,15 @@ def render_markdown(report: OptimizationReport) -> str: + ("yes" if report.run.get("update_source") else "no (default)"), "", ]) + availability = report.audit.get("sdk_result_availability", {}) + lines.extend([ + "Fake and SDK modes perform complete AgentEvaluator-compatible reevaluation for baseline " + "and every candidate on both train and validation; optimizer aggregates are never used " + "as gate evidence.", + "", + ]) if report.run.get("mode") == "sdk": - availability = report.audit.get("sdk_result_availability", {}) lines.extend([ - "SDK mode uses OptimizeResult aggregate validation metrics. " - "Full train scores and full per-case validation deltas are not exposed by the SDK result.", - "", "SDK availability: " f"aggregate_validation_result={availability.get('aggregate_validation_result')}, " f"full_train_eval_result={availability.get('full_train_eval_result')}, " @@ -148,16 +278,10 @@ def render_markdown(report: OptimizationReport) -> str: "", ]) for decision in report.gate_decisions: - verdict = ( - decision.gate_status - if decision.gate_status != "applied" - else ("accepted" if decision.accepted else "rejected") - ) + verdict = "accepted" if decision.accepted else "rejected" lines.append(f"### {decision.candidate_id} ({verdict})") for reason in decision.reasons: lines.append(f"- {reason}") - if decision.not_applied_checks: - lines.append(f"- not applied checks: {', '.join(decision.not_applied_checks)}") lines.append("") lines.extend([ @@ -170,7 +294,7 @@ def render_markdown(report: OptimizationReport) -> str: for record in report.candidates: candidate: CandidatePrompt = record["candidate"] gate = decision_by_id[candidate.candidate_id] - verdict = gate.gate_status if gate.gate_status != "applied" else ("accept" if gate.accepted else "reject") + verdict = "accept" if gate.accepted else "reject" lines.append( f"| {candidate.candidate_id} | {record['train_result'].score:.3f} | " f"{record['validation_result'].score:.3f} | {verdict} |" @@ -252,8 +376,6 @@ def render_markdown(report: OptimizationReport) -> str: def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None: run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) run_dir = output_path / "runs" / run_id - if run_dir.exists() and report.run.get("mode") == "fake": - shutil.rmtree(run_dir) run_dir.mkdir(parents=True, exist_ok=True) input_paths = report.audit.get("input_paths", {}) @@ -280,14 +402,12 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None candidate_name = _safe_artifact_name(candidate.candidate_id) candidate_dir = prompt_dir / candidate_name candidate_dir.mkdir(exist_ok=True) - best_prompts = report.audit.get("sdk_result_summary", {}).get("best_prompts", {}) - if report.run.get("mode") == "sdk" and isinstance(best_prompts, dict) and best_prompts: - for field_name, prompt_text in best_prompts.items(): - field_artifact = _safe_artifact_name(str(field_name)) - (candidate_dir / f"{field_artifact}.txt").write_text(str(prompt_text), encoding="utf-8") - (candidate_dir / "bundle.txt").write_text(candidate.prompt, encoding="utf-8") - else: - (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8") + for field_name, prompt_text in candidate.bundle().items(): + field_artifact = _safe_artifact_name(str(field_name)) + (candidate_dir / f"{field_artifact}.txt").write_text( + prompt_text, + encoding="utf-8", + ) (diffs_dir / f"{candidate_name}.diff").write_text(candidate.prompt_diff, encoding="utf-8") for split_name in ("train_result", "validation_result"): split_result = record[split_name] diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 29f38ded..2279501c 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -1,13 +1,12 @@ -"""Run the deterministic Evaluation + Optimization closed-loop example.""" +"""Run the shared asynchronous Evaluation + Optimization pipeline.""" from __future__ import annotations import argparse -import hashlib +import asyncio import json -import math import re -import shlex +import secrets import sys import tempfile from datetime import datetime @@ -15,28 +14,29 @@ from pathlib import Path from typing import Any + HERE = Path(__file__).resolve().parent if str(HERE) not in sys.path: sys.path.insert(0, str(HERE)) -from eval_loop.backends import FakeBackend -from eval_loop.backends import SDKBackend -from eval_loop.config import _finite_number -from eval_loop.config import validate_inputs -from eval_loop.gate import AcceptanceGate -from eval_loop.loader import load_eval_cases -from eval_loop.loader import load_optimizer_config -from eval_loop.loader import load_prompt -from eval_loop.loader import sha256_file -from eval_loop.loader import stable_config_hash -from eval_loop.report import REPRODUCIBILITY_COMMAND -from eval_loop.report import build_report -from eval_loop.report import compute_case_deltas -from eval_loop.report import write_reports -from eval_loop.schemas import CandidatePrompt -from eval_loop.schemas import EvalResult -from eval_loop.schemas import GateDecision -from eval_loop.schemas import OptimizationReport +try: # Package import during tests. + from .eval_loop.backends import FakeBackend + from .eval_loop.backends import SDKBackend + from .eval_loop.config import _parse_gate_config + from .eval_loop.config import parse_optimizer_config + from .eval_loop.loader import read_json + from .eval_loop.pipeline import PipelineRequest + from .eval_loop.pipeline import execute_pipeline + from .eval_loop.schemas import OptimizationReport +except ImportError: # Direct script execution. + from eval_loop.backends import FakeBackend + from eval_loop.backends import SDKBackend + from eval_loop.config import _parse_gate_config + from eval_loop.config import parse_optimizer_config + from eval_loop.loader import read_json + from eval_loop.pipeline import PipelineRequest + from eval_loop.pipeline import execute_pipeline + from eval_loop.schemas import OptimizationReport DEFAULT_TRAIN = HERE / "data" / "train.evalset.json" @@ -48,7 +48,7 @@ RUN_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") -def run_pipeline( +async def run_pipeline_async( *, train_path: str | Path = DEFAULT_TRAIN, val_path: str | Path = DEFAULT_VAL, @@ -64,647 +64,182 @@ def run_pipeline( gate_config_path: str | Path | None = None, target_prompts: list[str] | None = None, run_id: str | None = None, + backend: Any | None = None, ) -> OptimizationReport: - """Run baseline eval, fake optimization, validation gate, and reports.""" + """Build dependencies and await the backend-neutral orchestration core.""" - if mode not in {"fake", "sdk"}: - raise ValueError("field 'mode' must be one of: fake, sdk") - if run_id is not None: - run_id = validate_run_id(run_id) - if mode == "fake" and (not fake_model or not fake_judge): - raise ValueError( - "fake mode requires fake_model=True and fake_judge=True. Pass --fake-model --fake-judge " - "or use --mode sdk with --sdk-call-agent module:function." - ) - - if mode == "sdk": - optimizer_config_dict = _read_json_object_for_audit(optimizer_config_path) - baseline_prompt = load_prompt(prompt_path) - sdk_artifact_dir = Path(output_dir) / "sdk_optimizer" - wrapper_gate_config = _load_sdk_gate_config(gate_config_path) - target_prompt_paths = _parse_target_prompt_paths(target_prompts, default_prompt_path=prompt_path) - sdk_backend = SDKBackend( - prompt_path=prompt_path, - call_agent_path=sdk_call_agent, - update_source=update_source, - target_prompt_paths=target_prompt_paths, - ) - candidates = sdk_backend.optimize( - baseline_prompt=baseline_prompt, - train_path=train_path, - val_path=val_path, - optimizer_config_path=optimizer_config_path, - output_dir=sdk_artifact_dir, - ) - report = _build_sdk_report( - candidates=candidates, - sdk_backend=sdk_backend, - train_path=train_path, - val_path=val_path, - optimizer_config_path=optimizer_config_path, - prompt_path=prompt_path, - output_dir=output_dir, - trace=trace, - update_source=update_source, - train_case_count=_count_cases(train_path), - validation_case_count=_count_cases(val_path), - optimizer_config_dict=optimizer_config_dict, - gate_config=wrapper_gate_config, - gate_config_path=gate_config_path, - target_prompt_paths=target_prompt_paths, - sdk_call_agent=sdk_call_agent, - run_id=run_id, - ) - if run_id is None: - _resolve_default_sdk_run_id_collision(report, output_dir) - write_reports(report, output_dir) - return report - - optimizer_config = load_optimizer_config(optimizer_config_path) - train_cases = load_eval_cases(train_path, split="train") - validation_cases = load_eval_cases(val_path, split="validation") - validate_inputs( - train_path=train_path, - val_path=val_path, - optimizer_config_path=optimizer_config_path, - train_cases=train_cases, - validation_cases=validation_cases, - config=optimizer_config, - ) - - seed = optimizer_config.seed - baseline_prompt = load_prompt(prompt_path) - backend = FakeBackend(seed=seed, trace_enabled=trace) - - baseline = CandidatePrompt( - candidate_id="baseline", - prompt=baseline_prompt, - rationale="Prompt source file before optimization.", - prompt_diff="", - ) - baseline_train = backend.evaluate( - prompt_id=baseline.candidate_id, - prompt=baseline.prompt, - cases=train_cases, - split="train", - ) - baseline_validation = backend.evaluate( - prompt_id=baseline.candidate_id, - prompt=baseline.prompt, - cases=validation_cases, - split="validation", - ) - - candidates = backend.optimize( - baseline_prompt=baseline_prompt, - train_path=train_path, - val_path=val_path, - optimizer_config_path=optimizer_config_path, - output_dir=output_dir, - ) - gate = AcceptanceGate(optimizer_config.gate.to_dict()) - - candidate_records: list[dict[str, Any]] = [] - all_deltas = [] - gate_decisions = [] - cumulative_cost = round(baseline_train.cost + baseline_validation.cost, 6) - for candidate in candidates: - train_result = backend.evaluate( - prompt_id=candidate.candidate_id, - prompt=candidate.prompt, - cases=train_cases, - split="train", - ) - validation_result = backend.evaluate( - prompt_id=candidate.candidate_id, - prompt=candidate.prompt, - cases=validation_cases, - split="validation", - ) - deltas = compute_case_deltas( - candidate_id=candidate.candidate_id, - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_train=train_result, - candidate_validation=validation_result, - ) - decision = gate.decide( - candidate_id=candidate.candidate_id, - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_train=train_result, - candidate_validation=validation_result, - deltas=deltas, - cumulative_cost=cumulative_cost, - ) - candidate_records.append({ - "candidate": candidate, - "train_result": train_result, - "validation_result": validation_result, - }) - all_deltas.extend(deltas) - gate_decisions.append(decision) - cumulative_cost = decision.total_run_cost - - selected_candidate = _select_candidate(candidate_records, gate_decisions) - input_hashes = _input_hashes( + request, selected_backend = build_pipeline_request_and_backend( train_path=train_path, val_path=val_path, optimizer_config_path=optimizer_config_path, prompt_path=prompt_path, + output_dir=output_dir, + mode=mode, + fake_model=fake_model, + fake_judge=fake_judge, + trace=trace, + sdk_call_agent=sdk_call_agent, + update_source=update_source, + gate_config_path=gate_config_path, + target_prompts=target_prompts, + run_id=run_id, + backend=backend, ) - audit = _build_audit( - seed=seed, - config_hash=stable_config_hash(optimizer_config.to_dict()), - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_records=candidate_records, - candidates=candidates, - input_hashes=input_hashes, - input_paths={ - "train": str(train_path), - "validation": str(val_path), - "optimizer": str(optimizer_config_path), - "prompt": str(prompt_path), - }, - ) - run = { - "run_id": f"eval_optimize_loop_seed_{seed}", - "mode": mode, - "fake_model": fake_model, - "fake_judge": fake_judge, - "trace_enabled": trace, - "train_cases": len(train_cases), - "validation_cases": len(validation_cases), - "update_source": update_source, - "reproducibility_command": REPRODUCIBILITY_COMMAND, - "paths": { - "train": str(train_path), - "validation": str(val_path), - "optimizer": str(optimizer_config_path), - "prompt": str(prompt_path), - }, - "prompt_source": str(prompt_path), - } - report = build_report( - run=run, - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_records=candidate_records, - per_case_deltas=all_deltas, - gate_decisions=gate_decisions, - selected_candidate=selected_candidate, - audit=audit, - ) - write_reports(report, output_dir) - return report - - -def _select_candidate(candidate_records: list[dict[str, Any]], gate_decisions: list) -> str | None: - decisions_by_id = {decision.candidate_id: decision for decision in gate_decisions} - accepted = [] - for index, record in enumerate(candidate_records): - candidate = record["candidate"] - decision = decisions_by_id[candidate.candidate_id] - if decision.accepted: - accepted.append((index, record)) - if not accepted: - return None - index, record = max( - accepted, - key=lambda item: ( - item[1]["validation_result"].score, - item[1]["train_result"].score, - -item[0], - ), + return await execute_pipeline( + request, + evaluator=selected_backend, + optimizer=selected_backend, ) - return record["candidate"].candidate_id -def _build_audit( - *, - seed: int, - config_hash: str, - baseline_train, - baseline_validation, - candidate_records: list[dict[str, Any]], - candidates: list[CandidatePrompt], - input_hashes: dict[str, str], - input_paths: dict[str, str], -) -> dict[str, Any]: - baseline_cost = round(baseline_train.cost + baseline_validation.cost, 6) - candidate_costs = { - record["candidate"].candidate_id: round(record["train_result"].cost + record["validation_result"].cost, 6) - for record in candidate_records - } - total_cost = round(baseline_cost + sum(candidate_costs.values()), 6) - candidate_prompt_hashes = { - candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() - for candidate in candidates - } - return { - "seed": seed, - "duration_seconds": 0.0, - "config_hash": config_hash, - "input_hashes": input_hashes, - "input_paths": input_paths, - "prompt_hash": input_hashes["prompt"], - "candidate_prompt_hashes": candidate_prompt_hashes, - "total_run_cost": total_cost, - "cost": { - "baseline": baseline_cost, - "candidates": candidate_costs, - "total": total_cost, - }, - "candidate_prompts": { - candidate.candidate_id: { - "rationale": candidate.rationale, - "prompt": candidate.prompt, - "prompt_diff": candidate.prompt_diff, - } - for candidate in candidates - }, - "prompt_diffs": { - candidate.candidate_id: candidate.prompt_diff - for candidate in candidates - }, - "reproducibility_command": REPRODUCIBILITY_COMMAND, - } - - -def _build_sdk_report( +def run_pipeline( *, - candidates: list[CandidatePrompt], - sdk_backend: SDKBackend, - train_path: str | Path, - val_path: str | Path, - optimizer_config_path: str | Path, - prompt_path: str | Path, - output_dir: str | Path, - trace: bool, - update_source: bool, - train_case_count: int | None, - validation_case_count: int | None, - optimizer_config_dict: dict[str, Any], - gate_config: dict[str, Any], - gate_config_path: str | Path | None, - target_prompt_paths: dict[str, str | Path], - sdk_call_agent: str | None, - run_id: str | None, + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, + run_id: str | None = None, + backend: Any | None = None, ) -> OptimizationReport: - input_hashes = _input_hashes( - train_path=train_path, - val_path=val_path, - optimizer_config_path=optimizer_config_path, - prompt_path=prompt_path, - ) - sdk_summary = sdk_backend.last_result_summary or {} - baseline_pass_rate = _summary_float(sdk_summary, "baseline_pass_rate", 0.0, required=True) - best_pass_rate = _summary_float(sdk_summary, "best_pass_rate", baseline_pass_rate, required=True) - pass_rate_improvement = _summary_float( - sdk_summary, - "pass_rate_improvement", - best_pass_rate - baseline_pass_rate, - required=True, - ) - total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) - duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0) - effective_run_id = run_id or _default_sdk_run_id(sdk_summary) - target_prompt_hashes = { - name: sha256_file(path) - for name, path in target_prompt_paths.items() - } - input_hashes["target_prompts"] = target_prompt_hashes - if gate_config_path: - input_hashes["gate_config"] = sha256_file(gate_config_path) - availability = { - "aggregate_validation_result": True, - "full_train_eval_result": False, - "full_per_case_validation_delta": False, - } - score_explanation = ( - "SDK mode uses OptimizeResult aggregate validation metrics. " - "The full train EvalResult compatibility field is unavailable and keeps score 0.0; " - "full per-case validation deltas are unavailable and listed in not_applied_checks." - ) + """Synchronous compatibility facade for callers without an active loop.""" - baseline_train = EvalResult(prompt_id="baseline", split="train", score=0.0, passed=False, cost=0.0, cases=[]) - baseline_validation = EvalResult( - prompt_id="baseline", - split="validation", - score=baseline_pass_rate, - passed=baseline_pass_rate >= 1.0, - cost=0.0, - cases=[], - ) - candidate_records = [ - { - "candidate": candidate, - "train_result": EvalResult( - prompt_id=candidate.candidate_id, - split="train", - score=0.0, - passed=False, - cost=0.0, - cases=[], - ), - "validation_result": EvalResult( - prompt_id=candidate.candidate_id, - split="validation", - score=best_pass_rate, - passed=best_pass_rate >= baseline_pass_rate, - cost=total_llm_cost, - cases=[], - ), - "gate_status": "partial_applied", - "gate_not_applied_reason": "SDK OptimizeResult exposes aggregate scores but not full per-case deltas", - "sdk_result_summary": sdk_summary, - } - for candidate in candidates - ] - prompt_hashes = { - candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() - for candidate in candidates - } - field_prompt_hashes = _candidate_prompt_hashes_by_field(candidates, sdk_summary) - audit = { - "seed": None, - "duration_seconds": duration_seconds, - "config_hash": stable_config_hash(optimizer_config_dict), - "input_hashes": input_hashes, - "input_paths": { - "train": str(train_path), - "validation": str(val_path), - "optimizer": str(optimizer_config_path), - "prompt": str(prompt_path), - }, - "prompt_hash": input_hashes["prompt"], - "candidate_prompt_hashes": prompt_hashes, - "candidate_prompt_hashes_by_field": field_prompt_hashes, - "target_prompt_hashes": target_prompt_hashes, - "sdk_result_availability": availability, - "sdk_score_explanation": score_explanation, - "wrapper_gate_config": dict(gate_config), - "wrapper_gate_config_path": str(gate_config_path) if gate_config_path else None, - "total_run_cost": total_llm_cost, - "cost": { - "baseline": 0.0, - "candidates": {candidate.candidate_id: total_llm_cost for candidate in candidates}, - "total": total_llm_cost, - }, - "candidate_prompts": { - candidate.candidate_id: { - "rationale": candidate.rationale, - "prompt": candidate.prompt, - "prompt_diff": candidate.prompt_diff, - } - for candidate in candidates - }, - "prompt_diffs": {candidate.candidate_id: candidate.prompt_diff for candidate in candidates}, - "sdk_artifact_dir": sdk_backend.last_artifact_dir or str(Path(output_dir) / "sdk_optimizer"), - "sdk_result_summary": sdk_summary, - "reproducibility_command": _sdk_reproducibility_command( + if _has_running_loop(): + raise ValueError( + "run_pipeline() cannot run while an event loop is active; " + "await run_pipeline_async(...) instead." + ) + return asyncio.run( + run_pipeline_async( train_path=train_path, val_path=val_path, optimizer_config_path=optimizer_config_path, prompt_path=prompt_path, output_dir=output_dir, + mode=mode, + fake_model=fake_model, + fake_judge=fake_judge, + trace=trace, + sdk_call_agent=sdk_call_agent, update_source=update_source, gate_config_path=gate_config_path, - target_prompt_paths=target_prompt_paths, - sdk_call_agent=sdk_call_agent, + target_prompts=target_prompts, run_id=run_id, - ), - } - run = { - "run_id": effective_run_id, - "mode": "sdk", - "fake_model": False, - "fake_judge": False, - "trace_enabled": trace, - "train_cases": train_case_count, - "validation_cases": validation_case_count, - "update_source": update_source, - "sdk_artifact_dir": audit["sdk_artifact_dir"], - "sdk_availability": availability, - "wrapper_gate_config_path": str(gate_config_path) if gate_config_path else None, - "reproducibility_command": audit["reproducibility_command"], - "paths": { - "train": str(train_path), - "validation": str(val_path), - "optimizer": str(optimizer_config_path), - "prompt": str(prompt_path), - }, - "target_prompts": {name: str(path) for name, path in target_prompt_paths.items()}, - "prompt_source": str(prompt_path), - } - gate_decisions = [ - _sdk_gate_decision( - candidate_id=candidate.candidate_id, - sdk_summary=sdk_summary, - gate_config=gate_config, + backend=backend, ) - for candidate in candidates - ] - selected_candidate = None - if candidates and gate_decisions and gate_decisions[0].accepted: - selected_candidate = candidates[0].candidate_id - return build_report( - run=run, - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_records=candidate_records, - per_case_deltas=[], - gate_decisions=gate_decisions, - selected_candidate=selected_candidate, - audit=audit, ) -def _sdk_reproducibility_command( +def build_pipeline_request_and_backend( *, - train_path: str | Path, - val_path: str | Path, - optimizer_config_path: str | Path, - prompt_path: str | Path, - output_dir: str | Path, - update_source: bool, - gate_config_path: str | Path | None, - target_prompt_paths: dict[str, str | Path], - sdk_call_agent: str | None, - run_id: str | None, -) -> str: - parts = [ - "python", - "examples/optimization/eval_optimize_loop/run_pipeline.py", - "--mode", - "sdk", - "--train", - str(train_path), - "--val", - str(val_path), - "--optimizer-config", - str(optimizer_config_path), - "--prompt", - str(prompt_path), - "--output-dir", - str(output_dir), - "--sdk-call-agent", - sdk_call_agent or "", - ] - for name, path in target_prompt_paths.items(): - if not (name == "system_prompt" and Path(path) == Path(prompt_path) and len(target_prompt_paths) == 1): - parts.extend(["--target-prompt", f"{name}={path}"]) - if gate_config_path: - parts.extend(["--gate-config", str(gate_config_path)]) - if run_id: - parts.extend(["--run-id", run_id]) - if update_source: - parts.append("--update-source") - return " ".join(shlex.quote(part) for part in parts) - - -def _sdk_gate_decision( - *, - candidate_id: str, - sdk_summary: dict[str, Any], - gate_config: dict[str, Any], -) -> GateDecision: - status = str(sdk_summary.get("status") or "UNKNOWN") - improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0, required=True) - total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) - min_improvement = gate_config["min_val_score_improvement"] - max_cost = gate_config["max_total_cost"] - - reasons: list[str] = [] - accepted = True - if status != "SUCCEEDED": - accepted = False - reasons.append(f"reject: SDK optimizer status {status} is not SUCCEEDED") - else: - reasons.append("accept: SDK optimizer status is SUCCEEDED") + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, + run_id: str | None = None, + backend: Any | None = None, +) -> tuple[PipelineRequest, Any]: + """Validate wrapper inputs and construct one backend for both protocols.""" - if improvement < min_improvement: - accepted = False - reasons.append( - f"reject: validation improvement {improvement:.3f} is below threshold {min_improvement:.3f}" - ) - else: - reasons.append( - f"accept: validation improvement {improvement:.3f} meets threshold {min_improvement:.3f}" + if mode not in {"fake", "sdk"}: + raise ValueError("field 'mode' must be one of: fake, sdk") + if mode == "fake" and (not fake_model or not fake_judge): + raise ValueError( + "fake mode requires fake_model=True and fake_judge=True. Pass --fake-model " + "--fake-judge or use --mode sdk with --sdk-call-agent module:function." ) - if max_cost is not None: - if total_cost > max_cost: - accepted = False - reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}") + target_prompt_paths = _parse_target_prompt_paths( + target_prompts, + default_prompt_path=prompt_path, + ) + effective_run_id = validate_run_id(run_id) if run_id is not None else _default_run_id(mode) + + if mode == "fake": + optimizer_payload = read_json(optimizer_config_path) + optimizer_config = parse_optimizer_config( + optimizer_payload, + path=optimizer_config_path, + ) + gate_config = ( + _load_sdk_gate_config(gate_config_path) + if gate_config_path is not None + else optimizer_config.gate.to_dict() + ) + selected_backend = backend or FakeBackend( + seed=optimizer_config.seed, + trace_enabled=trace, + ) + else: + gate_config = _load_sdk_gate_config(gate_config_path) + if backend is None: + if not sdk_call_agent: + raise ValueError("sdk mode requires --sdk-call-agent module:function") + selected_backend = SDKBackend( + prompt_path=target_prompt_paths.get("system_prompt", prompt_path), + call_agent_path=sdk_call_agent, + target_prompt_paths=target_prompt_paths, + ) else: - reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}") - - if accepted: - reasons.append("accept: SDK aggregate gate passed") - - return GateDecision( - candidate_id=candidate_id, - accepted=accepted, - reasons=reasons, - train_score_delta=0.0, - validation_score_delta=improvement, - new_hard_failures=[], - protected_regressions=[], - validation_new_failures=[], - excessive_score_drops=[], - overfit_detected=False, - candidate_cost=total_cost, - cumulative_cost=0.0, - total_run_cost=total_cost, - cost=total_cost, - gate_status="partial_applied", - gate_not_applied_reason="SDK OptimizeResult exposes aggregate scores but not full per-case deltas", - not_applied_checks=[ - "per_case_delta", - "protected_regression", - "new_hard_failure", - "max_score_drop_per_case", - ], + selected_backend = backend + + request = PipelineRequest( + train_path=Path(train_path), + validation_path=Path(val_path), + optimizer_config_path=Path(optimizer_config_path), + output_dir=Path(output_dir), + target_prompt_paths=target_prompt_paths, + gate_config=gate_config, + trace=trace, + update_source=update_source, + mode=mode, + run_id=effective_run_id, + sdk_call_agent=sdk_call_agent, + gate_config_path=Path(gate_config_path) if gate_config_path is not None else None, ) + return request, selected_backend def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, Any]: + """Load the independent wrapper gate with the full strict GateConfig schema.""" + if gate_config_path is None: gate_payload: dict[str, Any] = {} path_text = "--gate-config" else: - payload = _read_json_object_for_audit(gate_config_path) + try: + payload = json.loads(Path(gate_config_path).read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + raise ValueError(f"--gate-config {gate_config_path}: invalid JSON: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError(f"--gate-config {gate_config_path}: expected a JSON object") gate_payload = payload.get("gate", payload) path_text = str(gate_config_path) if gate_payload is None: gate_payload = {} if not isinstance(gate_payload, dict): raise ValueError(f"{path_text}: field 'gate' must be an object when present") - - min_improvement = gate_payload.get("min_val_score_improvement", 0.01) - max_cost = gate_payload.get("max_total_cost", 1.0) - min_improvement = _finite_number( - min_improvement, - f"--gate-config {path_text}: field 'gate.min_val_score_improvement'", - 0.0, - 1.0, - ) - if max_cost is not None: - max_cost = _finite_number( - max_cost, - f"--gate-config {path_text}: field 'gate.max_total_cost'", - 0.0, - ) - return { - "min_val_score_improvement": min_improvement, - "max_total_cost": max_cost, - } - - -def _summary_float(summary: dict[str, Any], key: str, default: float, *, required: bool = False) -> float: - value = summary.get(key, default) - if value is None: - return default - if isinstance(value, bool): - if required: - raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") - return default - try: - parsed = float(value) - except (TypeError, ValueError): - if required: - raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") - return default - if not math.isfinite(parsed): - if required: - raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") - return default - return parsed - - -def _default_sdk_run_id(sdk_summary: dict[str, Any]) -> str: - started_at = sdk_summary.get("started_at") - if isinstance(started_at, str) and started_at.strip(): - source = started_at.strip() - try: - normalized = source[:-1] + "+00:00" if source.endswith("Z") else source - parsed = datetime.fromisoformat(normalized) - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return "eval_optimize_loop_sdk_" + parsed.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - except ValueError: - pass - else: - return "eval_optimize_loop_sdk_" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - safe = [] - for char in source: - if char.isalnum() or char in {"-", "_"}: - safe.append(char) - else: - safe.append("-") - return "eval_optimize_loop_sdk_" + "".join(safe).strip("-") + return _parse_gate_config( + gate_payload, + path=f"--gate-config {path_text}", + ).to_dict() def _parse_target_prompt_paths( @@ -712,18 +247,21 @@ def _parse_target_prompt_paths( *, default_prompt_path: str | Path, ) -> dict[str, str | Path]: + """Parse, resolve, and de-duplicate name=path target prompt arguments.""" + if not target_prompts: - return {"system_prompt": default_prompt_path} + return {"system_prompt": Path(default_prompt_path).resolve()} parsed: dict[str, str | Path] = {} resolved_paths: set[Path] = set() for item in target_prompts: if "=" not in item: raise ValueError("--target-prompt must use name=path format") - name, path = item.split("=", 1) - path = path.strip() + name, raw_path = item.split("=", 1) + path = raw_path.strip() if not TARGET_PROMPT_FIELD_RE.fullmatch(name): raise ValueError( - f"--target-prompt field name {name!r} is invalid; use /^[A-Za-z_][A-Za-z0-9_]*$/" + f"--target-prompt field name {name!r} is invalid; " + "use /^[A-Za-z_][A-Za-z0-9_]*$/" ) if not path: raise ValueError("--target-prompt must use non-empty name=path values") @@ -733,11 +271,13 @@ def _parse_target_prompt_paths( if resolved_path in resolved_paths: raise ValueError("--target-prompt fields must not reference the same resolved file") resolved_paths.add(resolved_path) - parsed[name] = Path(path) + parsed[name] = resolved_path return parsed def validate_run_id(run_id: str) -> str: + """Validate run IDs before using them as artifact directory names.""" + if not isinstance(run_id, str): raise ValueError(f"--run-id value {run_id!r} must be a string") if run_id in {"", ".", ".."} or not RUN_ID_RE.fullmatch(run_id): @@ -745,91 +285,46 @@ def validate_run_id(run_id: str) -> str: return run_id -def _resolve_default_sdk_run_id_collision(report: OptimizationReport, output_dir: str | Path) -> None: - base_run_id = str(report.run.get("run_id") or "") - validate_run_id(base_run_id) - run_root = Path(output_dir) / "runs" - candidate = base_run_id - suffix = 1 - while (run_root / candidate).exists(): - candidate = f"{base_run_id}-{suffix}" - suffix += 1 - report.run["run_id"] = candidate - - -def _candidate_prompt_hashes_by_field( - candidates: list[CandidatePrompt], - sdk_summary: dict[str, Any], -) -> dict[str, dict[str, str]]: - best_prompts = sdk_summary.get("best_prompts") - if not isinstance(best_prompts, dict): - return {} - return { - candidate.candidate_id: { - str(name): hashlib.sha256(str(prompt).encode("utf-8")).hexdigest() - for name, prompt in best_prompts.items() - } - for candidate in candidates - } +def _default_run_id(mode: str) -> str: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + return f"eval_optimize_loop_{mode}_{timestamp}_{secrets.token_hex(3)}" + + +def _has_running_loop() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--train", default=str(DEFAULT_TRAIN), help="Path to train.evalset.json") parser.add_argument("--val", default=str(DEFAULT_VAL), help="Path to val.evalset.json") - parser.add_argument("--optimizer-config", default=str(DEFAULT_OPTIMIZER_CONFIG), help="Path to optimizer.json") + parser.add_argument( + "--optimizer-config", + default=str(DEFAULT_OPTIMIZER_CONFIG), + help="Path to optimizer.json", + ) parser.add_argument("--prompt", default=str(DEFAULT_PROMPT), help="Path to baseline system prompt") parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR), help="Directory for runtime reports") parser.add_argument("--mode", choices=("fake", "sdk"), default="fake", help="Backend mode") parser.add_argument("--fake-model", action="store_true", help="Use deterministic fake model") parser.add_argument("--fake-judge", action="store_true", help="Use deterministic fake judge") - parser.add_argument("--trace", action="store_true", help="Persist fake model/judge trace details per case") + parser.add_argument("--trace", action="store_true", help="Persist evaluator trace details per case") parser.add_argument("--sdk-call-agent", help="Async call_agent target for SDK mode, as module:function") - parser.add_argument("--update-source", action="store_true", help="Allow SDK optimizer to write back source prompt") - parser.add_argument("--gate-config", help="Wrapper gate config for SDK mode; separate from SDK optimizer config") + parser.add_argument("--update-source", action="store_true", help="Write the accepted prompt after audit") + parser.add_argument("--gate-config", help="Independent wrapper gate configuration") parser.add_argument( "--target-prompt", action="append", - help="SDK target prompt path in name=path format. May be repeated. Defaults to system_prompt=--prompt.", + help="Target prompt as name=path; may be repeated and defaults to system_prompt=--prompt.", ) - parser.add_argument("--run-id", help="Optional report/audit run id. Fake mode keeps its deterministic default.") + parser.add_argument("--run-id", help="Optional stable report/audit run ID") return parser.parse_args(argv) -def _input_hashes( - *, - train_path: str | Path, - val_path: str | Path, - optimizer_config_path: str | Path, - prompt_path: str | Path, -) -> dict[str, str]: - return { - "train": sha256_file(train_path), - "validation": sha256_file(val_path), - "optimizer": sha256_file(optimizer_config_path), - "prompt": sha256_file(prompt_path), - } - - -def _count_cases(path: str | Path) -> int | None: - try: - payload = json.loads(Path(path).read_text(encoding="utf-8")) - except Exception: - return None - for key in ("cases", "eval_cases"): - cases = payload.get(key) if isinstance(payload, dict) else None - if isinstance(cases, list): - return len(cases) - return None - - -def _read_json_object_for_audit(path: str | Path) -> dict[str, Any]: - payload = json.loads(Path(path).read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise ValueError(f"{path}: optimizer config must be a JSON object") - return payload - - def main(argv: list[str] | None = None) -> OptimizationReport: args = parse_args(argv) report = run_pipeline( diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py index fc112caf..deb16a28 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_gate.py +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -3,6 +3,7 @@ from examples.optimization.eval_optimize_loop.eval_loop.gate import AcceptanceGate from examples.optimization.eval_optimize_loop.eval_loop.report import compute_case_deltas from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult @@ -26,6 +27,7 @@ def test_gate_rejects_train_improvement_with_validation_regression(): candidate_train=candidate_train, candidate_validation=candidate_val, deltas=deltas, + cost_summary=CostSummary(), ) assert not decision.accepted @@ -53,6 +55,7 @@ def test_gate_rejects_protected_case_regression(): candidate_train=candidate_train, candidate_validation=candidate_val, deltas=deltas, + cost_summary=CostSummary(), ) assert not decision.accepted @@ -80,6 +83,7 @@ def test_gate_accepts_safe_candidate(): candidate_train=candidate_train, candidate_validation=candidate_val, deltas=deltas, + cost_summary=CostSummary(), ) assert decision.accepted @@ -107,6 +111,7 @@ def test_gate_rejects_new_hard_failure_when_not_allowed(): candidate_train=candidate_train, candidate_validation=candidate_val, deltas=deltas, + cost_summary=CostSummary(), ) assert not decision.accepted @@ -114,6 +119,38 @@ def test_gate_rejects_new_hard_failure_when_not_allowed(): assert decision.validation_new_failures == ["val_a"] +def test_gate_rejects_soft_failure_that_becomes_hard_failure(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("soft_fail", 0.5), ("improved", 0.0), ("ok", 1.0)]) + candidate_val = _eval("candidate", "validation", [("soft_fail", 0.0), ("improved", 1.0), ("ok", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({ + "allow_new_hard_fail": False, + "max_score_drop_per_case": 1.0, + "min_val_score_improvement": 0.01, + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert not decision.accepted + assert decision.new_hard_failures == ["soft_fail"] + assert any("new hard failures" in reason for reason in decision.reasons) + + def test_gate_rejects_excessive_score_drop(): baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) @@ -134,6 +171,7 @@ def test_gate_rejects_excessive_score_drop(): candidate_train=candidate_train, candidate_validation=candidate_val, deltas=deltas, + cost_summary=CostSummary(), ) assert not decision.accepted @@ -160,6 +198,7 @@ def test_gate_rejects_cost_budget(): candidate_train=candidate_train, candidate_validation=candidate_val, deltas=deltas, + cost_summary=CostSummary(optimizer=0.001, total=0.001, complete=True), cumulative_cost=0.001, ) @@ -168,6 +207,166 @@ def test_gate_rejects_cost_budget(): assert decision.total_run_cost == decision.cumulative_cost + decision.candidate_cost +def test_gate_rejects_incomplete_cost_when_budget_is_configured(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + + decision = AcceptanceGate({"max_total_cost": 100.0}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ), + cost_summary=CostSummary(optimizer=0.5, total=0.5, complete=False), + cumulative_cost=0.502, + ) + + assert decision.accepted is False + assert any("cost_unavailable for configured max_total_cost" in reason for reason in decision.reasons) + assert decision.total_run_cost == 0.504 + + +def test_gate_skips_cost_completeness_check_when_budget_is_disabled(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + + decision = AcceptanceGate({"max_total_cost": None}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ), + cost_summary=CostSummary(optimizer=10.0, total=10.0, complete=False), + cumulative_cost=10.002, + ) + + assert decision.accepted is True + assert all("cost_unavailable" not in reason for reason in decision.reasons) + assert decision.total_run_cost == 10.004 + + +def test_gate_rejects_new_validation_failure_even_when_it_is_not_hard(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("new_soft", 1.0), ("up_a", 0.0), ("up_b", 0.0)]) + candidate_val = _eval("candidate", "validation", [("new_soft", 0.5), ("up_a", 1.0), ("up_b", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({ + "allow_new_hard_fail": True, + "max_score_drop_per_case": 1.0, + "max_total_cost": None, + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert decision.accepted is False + assert decision.validation_new_failures == ["new_soft"] + assert any("new validation failures" in reason for reason in decision.reasons) + + +def test_gate_does_not_add_cost_summary_total_twice(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + + decision = AcceptanceGate({"max_total_cost": 1.0}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ), + cost_summary=CostSummary(optimizer=0.4, total=0.4, complete=True), + cumulative_cost=0.6, + ) + + assert decision.total_run_cost == 0.602 + assert decision.accepted is True + + +def test_gate_rejects_duplicate_missing_and_empty_case_results(): + valid_train = _eval("baseline", "train", [("train_a", 0.0)]) + valid_candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + valid_val = _eval("baseline", "validation", [("val_a", 0.0)]) + valid_candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + scenarios = [ + ( + _eval("baseline", "train", [("train_a", 0.0), ("train_a", 0.0)]), + valid_candidate_train, + valid_val, + valid_candidate_val, + "duplicate case IDs", + ), + ( + valid_train, + _eval("candidate", "train", [("other", 1.0)]), + valid_val, + valid_candidate_val, + "case ID set mismatch", + ), + ( + _empty_eval("baseline", "train"), + _empty_eval("candidate", "train"), + valid_val, + valid_candidate_val, + "empty case results", + ), + ] + + for baseline_train, candidate_train, baseline_val, candidate_val, expected in scenarios: + decision = AcceptanceGate({"max_total_cost": None}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=[], + cost_summary=CostSummary(), + ) + assert decision.accepted is False + assert any(expected in reason for reason in decision.reasons) + assert decision.gate_status == "applied" + assert decision.not_applied_checks == [] + + def test_case_delta_types_are_classified(): baseline_train = _eval("baseline", "train", [("new_pass", 0.0), ("new_fail", 1.0)]) candidate_train = _eval("candidate", "train", [("new_pass", 1.0), ("new_fail", 0.0)]) @@ -210,3 +409,14 @@ def _eval(prompt_id: str, split: str, scores: list[tuple[str, float]]) -> EvalRe cost=0.001 * len(cases), cases=cases, ) + + +def _empty_eval(prompt_id: str, split: str) -> EvalResult: + return EvalResult( + prompt_id=prompt_id, + split=split, + score=0.0, + passed=False, + cost=0.0, + cases=[], + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py index 85247d7e..f9434728 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -69,12 +69,18 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): "candidate_prompt_hashes", "total_run_cost", } + assert payload["audit"]["seed"] == 91 + assert payload["audit"]["duration_seconds"] > 0 + assert payload["run"]["trace"] is True assert payload["run"]["reproducibility_command"].startswith("python examples/optimization") - assert payload["run"]["run_id"] == "eval_optimize_loop_seed_91" + assert payload["run"]["run_id"].startswith("eval_optimize_loop_fake_") assert payload["audit"]["total_run_cost"] == payload["audit"]["cost"]["total"] assert "candidate_001_overfit" in payload["audit"]["prompt_diffs"] assert "candidate_002_safe" in payload["audit"]["prompt_diffs"] assert payload["candidates"][0]["candidate"]["prompt_diff"].startswith("--- baseline_system_prompt.txt") + assert all(len(record["train_result"]["cases"]) == 3 for record in payload["candidates"]) + assert all(len(record["validation_result"]["cases"]) == 3 for record in payload["candidates"]) + assert payload["writeback"]["status"] == "not_requested" decisions = {item["candidate_id"]: item for item in payload["gate_decisions"]} assert not decisions["candidate_001_overfit"]["accepted"] @@ -96,7 +102,7 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert "Reproducibility" in markdown assert "Cost And Audit" in markdown - run_dir = output_dir / "runs" / "eval_optimize_loop_seed_91" + run_dir = output_dir / "runs" / payload["run"]["run_id"] assert (run_dir / "config.snapshot.json").is_file() assert (run_dir / "input_hashes.json").is_file() assert (run_dir / "candidate_prompts" / "candidate_001_overfit" / "system_prompt.txt").is_file() @@ -113,12 +119,23 @@ def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): run_pipeline(output_dir=first_dir, fake_model=True, fake_judge=True, trace=True) run_pipeline(output_dir=second_dir, fake_model=True, fake_judge=True, trace=True) - assert (first_dir / "optimization_report.json").read_text(encoding="utf-8") == ( - second_dir / "optimization_report.json" - ).read_text(encoding="utf-8") - assert (first_dir / "optimization_report.md").read_text(encoding="utf-8") == ( - second_dir / "optimization_report.md" - ).read_text(encoding="utf-8") + first_payload = json.loads((first_dir / "optimization_report.json").read_text(encoding="utf-8")) + second_payload = json.loads((second_dir / "optimization_report.json").read_text(encoding="utf-8")) + first_run_id = first_payload["run"]["run_id"] + second_run_id = second_payload["run"]["run_id"] + + assert _normalized_payload(first_payload) == _normalized_payload(second_payload) + first_markdown = (first_dir / "optimization_report.md").read_text(encoding="utf-8") + second_markdown = (second_dir / "optimization_report.md").read_text(encoding="utf-8") + normalized_first_markdown = first_markdown.replace(first_run_id, "").replace( + str(first_dir), + "", + ) + normalized_second_markdown = second_markdown.replace(second_run_id, "").replace( + str(second_dir), + "", + ) + assert normalized_first_markdown == normalized_second_markdown def test_pipeline_accepts_mode_fake_without_legacy_flags(tmp_path: Path): @@ -130,6 +147,12 @@ def test_pipeline_accepts_mode_fake_without_legacy_flags(tmp_path: Path): def test_pipeline_selected_candidate_is_null_when_all_candidates_rejected(tmp_path: Path): config_path = tmp_path / "optimizer.json" config = json.loads(Path(DEFAULT_OPTIMIZER_CONFIG).read_text(encoding="utf-8")) + config["gate"] = { + "allow_new_hard_fail": False, + "protected_case_ids": ["val_protected_yes_no"], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, + } config["gate"]["min_val_score_improvement"] = 1.0 config_path.write_text(json.dumps(config), encoding="utf-8") @@ -180,3 +203,12 @@ def test_fake_report_json_remains_strict_json(tmp_path: Path): assert "NaN" not in payload assert "Infinity" not in payload assert json.loads(payload)["selected_candidate"] == "candidate_002_safe" + + +def _normalized_payload(payload: dict) -> dict: + normalized = json.loads(json.dumps(payload)) + normalized["run"].pop("run_id", None) + normalized["run"].pop("reproducibility_command", None) + normalized["audit"].pop("duration_seconds", None) + normalized["audit"].pop("reproducibility_command", None) + return normalized diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py new file mode 100644 index 00000000..69a84321 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py @@ -0,0 +1,697 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.eval_loop.pipeline import PipelineRequest +from examples.optimization.eval_optimize_loop.eval_loop.pipeline import execute_pipeline +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CandidatePrompt +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import WritebackResult + + +class RecordingBackend: + def __init__( + self, + *, + candidates: list[CandidatePrompt], + results: dict[tuple[str, str], EvalResult], + optimization_cost: CostSummary | None = None, + ) -> None: + self.candidates = candidates + self.results = results + self.optimization_cost = optimization_cost or CostSummary() + self.calls: list[tuple[Any, ...]] = [] + self.failure_summary: dict[str, object] | None = None + + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + self.calls.append(("evaluate", prompt_id, split, dict(prompts), Path(artifact_dir), trace)) + return self.results[(prompt_id, split)] + + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + self.calls.append(("optimize", dict(baseline_prompts), Path(artifact_dir))) + self.failure_summary = failure_summary + return OptimizationResult( + candidates=self.candidates, + rounds=[], + cost=self.optimization_cost, + raw_summary={"status": "SUCCEEDED"}, + ) + + +@pytest.mark.asyncio +async def test_execute_pipeline_uses_train_only_evidence_and_full_candidate_reevaluation(tmp_path: Path): + request = _request(tmp_path, update_source=False) + backend = _safe_backend() + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert [(call[0], *call[1:3]) for call in backend.calls] == [ + ("evaluate", "baseline", "train"), + ("evaluate", "baseline", "validation"), + ("optimize", {"system_prompt": "baseline\n"}, tmp_path / "out" / "runs" / "ordered_run" / "optimizer"), + ("evaluate", "candidate_a", "train"), + ("evaluate", "candidate_a", "validation"), + ("evaluate", "candidate_b", "train"), + ("evaluate", "candidate_b", "validation"), + ] + assert backend.failure_summary is not None + assert backend.failure_summary["total_failed_cases"] == 1 + assert backend.failure_summary["by_category"] == {"train_only": 1} + assert "validation_only" not in str(backend.failure_summary) + assert report.selected_candidate == "candidate_a" + assert report.audit["duration_seconds"] > 0 + assert report.audit["config_snapshot"] == {"seed": 91} + assert report.audit["input_hashes"]["target_prompts"]["system_prompt"] == hashlib.sha256( + b"baseline\n" + ).hexdigest() + artifact_dirs = [call[4] for call in backend.calls if call[0] == "evaluate"] + assert len(artifact_dirs) == len(set(artifact_dirs)) == 6 + assert report.audit["sdk_result_availability"]["full_train_eval_result"] is True + assert report.audit["sdk_result_availability"]["full_per_case_validation_delta"] is True + + +@pytest.mark.asyncio +async def test_rejected_candidate_never_changes_source_when_writeback_requested(tmp_path: Path): + request = _request( + tmp_path, + update_source=True, + gate_config={"min_val_score_improvement": 0.5, "max_total_cost": None}, + ) + backend = _safe_backend(candidate_validation_score=0.0) + before = (tmp_path / "prompt.txt").read_bytes() + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.writeback.status == "rejected" + assert report.writeback.before_hashes == { + "system_prompt": hashlib.sha256(before).hexdigest(), + } + assert (tmp_path / "prompt.txt").read_bytes() == before + + +@pytest.mark.asyncio +async def test_accepted_candidate_is_not_written_when_writeback_not_requested(tmp_path: Path): + request = _request(tmp_path, update_source=False) + backend = _safe_backend() + before = (tmp_path / "prompt.txt").read_bytes() + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate == "candidate_a" + assert report.writeback.status == "not_requested" + assert report.writeback.before_hashes == { + "system_prompt": hashlib.sha256(before).hexdigest(), + } + assert (tmp_path / "prompt.txt").read_bytes() == before + + +@pytest.mark.asyncio +async def test_writeback_happens_only_after_prewrite_audit_succeeds( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend() + events: list[str] = [] + artifact_paths = object() + + def prepare(report, output_dir): + events.append("prepare") + assert report.selected_candidate == "candidate_a" + assert Path(output_dir) == tmp_path / "out" + return artifact_paths + + def commit(snapshot, prompts): + events.append("commit") + assert prompts == {"system_prompt": "safe prompt"} + return WritebackResult( + status="applied", + before_hashes=snapshot.hashes(), + after_hashes=snapshot.hashes(), + ) + + def finalize(report, paths): + events.append("finalize") + assert paths is artifact_paths + assert report.writeback.status == "applied" + + monkeypatch.setattr(pipeline_module, "prepare_run_artifacts", prepare) + monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", commit) + monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", finalize) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert events == ["prepare", "commit", "finalize"] + assert report.writeback.status == "applied" + + +@pytest.mark.asyncio +async def test_prepare_failure_prevents_source_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend() + before = (tmp_path / "prompt.txt").read_bytes() + commit_called = False + + def fail_prepare(report, output_dir): + raise OSError("audit unavailable") + + def forbidden_commit(snapshot, prompts): + nonlocal commit_called + commit_called = True + raise AssertionError("commit must not run") + + monkeypatch.setattr(pipeline_module, "prepare_run_artifacts", fail_prepare) + monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", forbidden_commit) + + with pytest.raises(OSError, match="audit unavailable"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert commit_called is False + assert (tmp_path / "prompt.txt").read_bytes() == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("max_total_cost", "accepted", "reason_fragment"), + [ + (1.0, False, "cost_unavailable"), + (None, True, None), + ], +) +async def test_incomplete_optimizer_cost_is_fail_closed_only_when_budget_configured( + tmp_path: Path, + max_total_cost: float | None, + accepted: bool, + reason_fragment: str | None, +): + request = _request( + tmp_path, + gate_config={ + "min_val_score_improvement": 0.0, + "max_score_drop_per_case": 1.0, + "max_total_cost": max_total_cost, + }, + ) + backend = _safe_backend( + candidate_count=1, + optimization_cost=CostSummary(optimizer=0.2, total=0.2, complete=False), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.gate_decisions[0].accepted is accepted + if reason_fragment is not None: + assert any(reason_fragment in reason for reason in report.gate_decisions[0].reasons) + else: + assert all("cost_unavailable" not in reason for reason in report.gate_decisions[0].reasons) + + +@pytest.mark.asyncio +async def test_optimizer_cost_is_counted_once_across_candidates(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend( + baseline_train_cost=0.1, + baseline_validation_cost=0.2, + candidate_train_cost=0.04, + candidate_validation_cost=0.05, + optimization_cost=CostSummary(optimizer=0.3, total=0.3, complete=True), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert [decision.total_run_cost for decision in report.gate_decisions] == [0.69, 0.78] + assert report.cost_summary == CostSummary( + optimizer=0.3, + evaluator=0.48, + agent=0.0, + total=0.78, + complete=True, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("baseline_case_ids", "candidate_case_ids", "reason_fragment"), + [ + (["a", "a"], ["a"], "duplicate case IDs"), + (["a"], ["a", "a"], "duplicate case IDs"), + (["a", "b"], ["a"], "case ID set mismatch"), + (["a"], ["a", "extra"], "case ID set mismatch"), + ([], [], "empty case results"), + ], +) +async def test_incomparable_case_results_are_rejected_without_delta_error( + tmp_path: Path, + baseline_case_ids: list[str], + candidate_case_ids: list[str], + reason_fragment: str, +): + request = _request(tmp_path) + candidate = CandidatePrompt("candidate_a", "safe prompt", "safe", "diff") + results = { + ("baseline", "train"): _eval("baseline", "train", baseline_case_ids, 0.0), + ("baseline", "validation"): _eval("baseline", "validation", baseline_case_ids, 0.0), + ("candidate_a", "train"): _eval("candidate_a", "train", candidate_case_ids, 1.0), + ("candidate_a", "validation"): _eval("candidate_a", "validation", candidate_case_ids, 1.0), + } + backend = RecordingBackend(candidates=[candidate], results=results) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.per_case_deltas == [] + assert report.gate_decisions[0].accepted is False + assert any(reason_fragment in reason for reason in report.gate_decisions[0].reasons) + + +@pytest.mark.asyncio +async def test_duplicate_candidate_ids_are_rejected(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + backend.candidates = [backend.candidates[0], backend.candidates[0]] + + with pytest.raises(ValueError, match="duplicate candidate_id"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("prompt_fields", "message"), + [ + ({"other_prompt": "candidate"}, "bundle fields"), + ({"system_prompt": ""}, "non-empty string"), + ], +) +async def test_candidate_bundle_must_match_target_snapshot( + tmp_path: Path, + prompt_fields: dict[str, str], + message: str, +): + request = _request(tmp_path) + candidate = CandidatePrompt( + candidate_id="candidate_a", + prompt="candidate", + rationale="test", + prompt_diff="diff", + prompt_fields=prompt_fields, + ) + backend = _safe_backend(candidate_count=0) + backend.candidates = [candidate] + + with pytest.raises(ValueError, match=message): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_target_prompt_hashes_cannot_overwrite_dataset_hashes(tmp_path: Path): + request = _request(tmp_path) + prompt_path = tmp_path / "reserved_name_prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + request = replace(request, target_prompt_paths={"train": prompt_path}) + backend = _safe_backend(candidate_count=1) + backend.candidates = [ + CandidatePrompt( + candidate_id="candidate_a", + prompt="safe prompt", + rationale="safe", + prompt_diff="diff", + prompt_fields={"train": "safe prompt"}, + ) + ] + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + expected_dataset_hash = hashlib.sha256(Path(request.train_path).read_bytes()).hexdigest() + expected_prompt_hash = hashlib.sha256(prompt_path.read_bytes()).hexdigest() + assert report.audit["input_hashes"]["train"] == expected_dataset_hash + assert report.audit["input_hashes"]["target_prompts"]["train"] == expected_prompt_hash + + +@pytest.mark.asyncio +async def test_finalize_failure_leaves_explicit_prepared_writeback_journal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + + monkeypatch.setattr( + pipeline_module, + "commit_prompt_bundle", + lambda snapshot, prompts: WritebackResult( + status="applied", + before_hashes=snapshot.hashes(), + after_hashes=snapshot.hashes(), + ), + ) + + def fail_finalize(report, paths): + raise OSError("final report unavailable") + + monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", fail_finalize) + + with pytest.raises(OSError, match="final report unavailable"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + run_dir = Path(request.output_dir) / "runs" / request.run_id + prewrite = json.loads((run_dir / "pre_write_report.json").read_text(encoding="utf-8")) + journal = json.loads((run_dir / "writeback_journal.json").read_text(encoding="utf-8")) + assert prewrite["audit"]["writeback_journal"]["state"] == "pending" + assert "pending" in prewrite["writeback"]["error"] + assert journal["state"] == "prepared" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("writeback_status", ["rolled_back", "rollback_failed"]) +async def test_failed_writeback_is_persisted_in_final_report_and_journal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + writeback_status: str, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + monkeypatch.setattr( + pipeline_module, + "commit_prompt_bundle", + lambda snapshot, prompts: WritebackResult( + status=writeback_status, + before_hashes=snapshot.hashes(), + after_hashes=snapshot.hashes(), + error="simulated commit failure", + ), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + run_dir = Path(request.output_dir) / "runs" / request.run_id + final_payload = json.loads((run_dir / "optimization_report.json").read_text(encoding="utf-8")) + writeback_payload = json.loads((run_dir / "writeback.json").read_text(encoding="utf-8")) + journal = json.loads((run_dir / "writeback_journal.json").read_text(encoding="utf-8")) + assert report.writeback.status == writeback_status + assert final_payload["writeback"]["status"] == writeback_status + assert writeback_payload["status"] == writeback_status + assert journal["state"] == writeback_status + + +@pytest.mark.asyncio +async def test_standard_evalset_protected_metadata_is_applied_to_gate(tmp_path: Path): + request = _request(tmp_path) + Path(request.validation_path).write_text( + """{ + "eval_set_id": "validation", + "eval_cases": [ + { + "eval_id": "protected", + "session_input": {"state": {"eval_optimize_protected": true}} + }, + { + "eval_id": "improved", + "session_input": {"state": {"eval_optimize_protected": false}} + } + ] +} +""", + encoding="utf-8", + ) + candidate = CandidatePrompt("candidate_a", "safe prompt", "safe", "diff") + results = { + ("baseline", "train"): _eval_scores("baseline", "train", [("train", 1.0)]), + ("baseline", "validation"): _eval_scores( + "baseline", + "validation", + [("protected", 0.8), ("improved", 0.0)], + ), + ("candidate_a", "train"): _eval_scores("candidate_a", "train", [("train", 1.0)]), + ("candidate_a", "validation"): _eval_scores( + "candidate_a", + "validation", + [("protected", 0.7), ("improved", 1.0)], + ), + } + backend = RecordingBackend(candidates=[candidate], results=results) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.gate_decisions[0].protected_regressions == ["protected"] + assert report.audit["gate_config_snapshot"]["protected_case_ids"] == ["protected"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("identity_error", "message"), + [ + ("prompt_id", "prompt_id"), + ("result_split", "EvalResult split"), + ("case_split", "CaseResult.*split"), + ], +) +async def test_backend_result_identity_must_match_requested_evaluation( + tmp_path: Path, + identity_error: str, + message: str, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + if identity_error == "prompt_id": + baseline_train = backend.results[("baseline", "train")] + backend.results[("baseline", "train")] = replace( + baseline_train, + prompt_id="wrong_prompt", + ) + elif identity_error == "result_split": + candidate_validation = backend.results[("candidate_a", "validation")] + backend.results[("candidate_a", "validation")] = replace( + candidate_validation, + split="train", + ) + else: + candidate_validation = backend.results[("candidate_a", "validation")] + backend.results[("candidate_a", "validation")] = replace( + candidate_validation, + cases=[replace(candidate_validation.cases[0], split="train")], + ) + + with pytest.raises(ValueError, match=message): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_async_wrapper_runs_inside_active_event_loop_with_injected_backend(tmp_path: Path): + from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline_async + + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + + report = await run_pipeline_async( + train_path=request.train_path, + val_path=request.validation_path, + optimizer_config_path=request.optimizer_config_path, + prompt_path=request.target_prompt_paths["system_prompt"], + output_dir=request.output_dir, + mode="fake", + trace=True, + run_id="async_wrapper_run", + backend=backend, + ) + + assert report.selected_candidate == "candidate_a" + assert report.run["run_id"] == "async_wrapper_run" + + +@pytest.mark.asyncio +async def test_sync_wrapper_rejects_active_event_loop_with_async_hint(tmp_path: Path): + from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="await run_pipeline_async"): + run_pipeline( + train_path=request.train_path, + val_path=request.validation_path, + optimizer_config_path=request.optimizer_config_path, + prompt_path=request.target_prompt_paths["system_prompt"], + output_dir=request.output_dir, + mode="fake", + run_id="sync_wrapper_run", + backend=backend, + ) + + +def _request( + tmp_path: Path, + *, + update_source: bool = False, + gate_config: dict[str, object] | None = None, +) -> PipelineRequest: + train_path = tmp_path / "train.evalset.json" + validation_path = tmp_path / "validation.evalset.json" + config_path = tmp_path / "optimizer.json" + prompt_path = tmp_path / "prompt.txt" + train_path.write_text('{"eval_cases": []}', encoding="utf-8") + validation_path.write_text('{"eval_cases": []}', encoding="utf-8") + config_path.write_text('{"seed": 91}', encoding="utf-8") + prompt_path.write_bytes(b"baseline\n") + return PipelineRequest( + train_path=train_path, + validation_path=validation_path, + optimizer_config_path=config_path, + output_dir=tmp_path / "out", + target_prompt_paths={"system_prompt": prompt_path}, + gate_config=gate_config or { + "min_val_score_improvement": 0.0, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 1.0, + "max_total_cost": None, + }, + trace=True, + update_source=update_source, + mode="fake", + run_id="ordered_run", + ) + + +def _safe_backend( + *, + candidate_count: int = 2, + candidate_validation_score: float = 1.0, + baseline_train_cost: float = 0.01, + baseline_validation_cost: float = 0.01, + candidate_train_cost: float = 0.01, + candidate_validation_cost: float = 0.01, + optimization_cost: CostSummary | None = None, +) -> RecordingBackend: + candidates = [ + CandidatePrompt("candidate_a", "safe prompt", "safe", "diff a"), + CandidatePrompt("candidate_b", "other prompt", "other", "diff b"), + ][:candidate_count] + results = { + ("baseline", "train"): _eval( + "baseline", + "train", + ["train_case"], + 0.0, + cost=baseline_train_cost, + failure_category="train_only", + ), + ("baseline", "validation"): _eval( + "baseline", + "validation", + ["validation_case"], + 0.0, + cost=baseline_validation_cost, + failure_category="validation_only", + ), + } + for candidate in candidates: + results[(candidate.candidate_id, "train")] = _eval( + candidate.candidate_id, + "train", + ["train_case"], + 1.0, + cost=candidate_train_cost, + ) + results[(candidate.candidate_id, "validation")] = _eval( + candidate.candidate_id, + "validation", + ["validation_case"], + candidate_validation_score, + cost=candidate_validation_cost, + ) + return RecordingBackend( + candidates=candidates, + results=results, + optimization_cost=optimization_cost, + ) + + +def _eval( + prompt_id: str, + split: str, + case_ids: list[str], + score: float, + *, + cost: float = 0.01, + failure_category: str | None = None, +) -> EvalResult: + cases = [ + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="output", + hard_failed=score <= 0.0, + failure_category=failure_category if score < 1.0 else None, + ) + for case_id in case_ids + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=score, + passed=bool(cases) and all(case.passed for case in cases), + cost=cost, + cases=cases, + ) + + +def _eval_scores( + prompt_id: str, + split: str, + scores: list[tuple[str, float]], +) -> EvalResult: + cases = [ + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="output", + hard_failed=score <= 0.0, + ) + for case_id, score in scores + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=round(sum(case.score for case in cases) / len(cases), 6), + passed=all(case.passed for case in cases), + cost=0.01, + cases=cases, + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 8d657061..4c8893d7 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -3,6 +3,7 @@ import builtins import inspect import json +import shlex import sys import types from pathlib import Path @@ -20,7 +21,6 @@ from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL from examples.optimization.eval_optimize_loop.run_pipeline import _parse_target_prompt_paths -from examples.optimization.eval_optimize_loop.run_pipeline import _sdk_gate_decision from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline @@ -1188,6 +1188,11 @@ async def test_sdk_backend_async_api_works_inside_active_event_loop(tmp_path: Pa def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, monkeypatch): calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + gate_path = _write_gate_config( + tmp_path, + min_val_score_improvement=0.01, + max_total_cost=None, + ) report = run_pipeline( mode="sdk", @@ -1197,6 +1202,7 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo prompt_path=DEFAULT_PROMPT, output_dir=tmp_path / "sdk_run", sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, run_id="sdk_test_run", ) @@ -1209,15 +1215,10 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert report.baseline_validation.score == 0.5 assert report.candidates[0]["validation_result"].score == 0.75 assert report.gate_decisions[0].validation_score_delta == 0.25 - assert report.gate_decisions[0].candidate_cost == 0.123 - assert report.gate_decisions[0].gate_status == "partial_applied" - assert report.gate_decisions[0].not_applied_checks == [ - "per_case_delta", - "protected_regression", - "new_hard_failure", - "max_score_drop_per_case", - ] - assert report.audit["duration_seconds"] == 12.3 + assert report.gate_decisions[0].candidate_cost == 0.0 + assert report.gate_decisions[0].gate_status == "applied" + assert report.gate_decisions[0].not_applied_checks == [] + assert report.audit["duration_seconds"] > 0 assert report.audit["total_run_cost"] == 0.123 assert report.audit["cost"]["total"] == 0.123 assert report.audit["sdk_result_summary"]["status"] == "SUCCEEDED" @@ -1232,21 +1233,24 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert report.audit["sdk_result_summary"]["rounds"][0]["validation_pass_rate"] == 0.75 assert report.audit["sdk_result_availability"] == { "aggregate_validation_result": True, - "full_train_eval_result": False, - "full_per_case_validation_delta": False, + "full_train_eval_result": True, + "full_per_case_validation_delta": True, } - assert "train EvalResult compatibility field is unavailable" in report.audit["sdk_score_explanation"] - assert "partial_applied" in payload - assert "sdk_best (partial_applied)" in markdown - assert "not applied checks: per_case_delta" in markdown - assert "SDK mode uses OptimizeResult aggregate validation metrics" in markdown - assert "fake_call_agent_module:call_agent" in report.run["reproducibility_command"] - assert "module:function" not in report.run["reproducibility_command"] + assert "partial_applied" not in payload + assert "sdk_best (accepted)" in markdown + assert "complete AgentEvaluator-compatible reevaluation" in markdown assert (output_dir / "runs" / "sdk_test_run" / "input_hashes.json").is_file() assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / "sdk_best.diff").is_file() assert calls["update_source"] is False - assert calls["output_dir"].endswith("sdk_optimizer") - assert report.run["sdk_artifact_dir"].endswith("sdk_optimizer") + assert calls["output_dir"].endswith("runs\\sdk_test_run\\optimizer") or calls[ + "output_dir" + ].endswith("runs/sdk_test_run/optimizer") + assert calls["evaluation_count"] == 4 + command = report.run["reproducibility_command"] + assert "--sdk-call-agent fake_call_agent_module:call_agent" in command + command_args = shlex.split(command) + gate_index = command_args.index("--gate-config") + assert command_args[gate_index + 1] == str(gate_path) def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp_path: Path, monkeypatch): @@ -1255,8 +1259,14 @@ def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp val_path = tmp_path / "sdk_val.evalset.json" optimizer_path = tmp_path / "sdk_optimizer.json" prompt_path = tmp_path / "system_prompt.txt" - train_path.write_text(json.dumps({"eval_cases": []}), encoding="utf-8") - val_path.write_text(json.dumps({"eval_cases": []}), encoding="utf-8") + train_path.write_text( + json.dumps(_sdk_evalset_payload([_sdk_eval_case_payload("sdk_train_case")])), + encoding="utf-8", + ) + val_path.write_text( + json.dumps(_sdk_evalset_payload([_sdk_eval_case_payload("sdk_val_case")])), + encoding="utf-8", + ) optimizer_path.write_text( json.dumps({"seed": "sdk-owned-seed", "optimize": {"algorithm": {"name": "gepa_reflective"}}}), encoding="utf-8", @@ -1271,14 +1281,22 @@ def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp prompt_path=prompt_path, output_dir=tmp_path / "sdk_run", sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=_write_gate_config( + tmp_path, + min_val_score_improvement=0.01, + max_total_cost=None, + ), ) assert report.run["mode"] == "sdk" - assert report.run["train_cases"] == 0 + assert report.run["train_cases"] == 1 assert report.selected_candidate == "sdk_best" -def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path, monkeypatch): +def test_run_pipeline_mode_sdk_default_run_id_uses_utc_timestamp_and_random_suffix( + tmp_path: Path, + monkeypatch, +): _install_fake_sdk( monkeypatch, best_prompt="optimized prompt", @@ -1295,12 +1313,13 @@ def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path sdk_call_agent="fake_call_agent_module:call_agent", ) - assert report.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z" + assert report.run["run_id"].startswith("eval_optimize_loop_sdk_") + assert report.run["run_id"] != "eval_optimize_loop_sdk_20260704T123456Z" assert (tmp_path / "sdk_run" / "runs" / report.run["run_id"]).is_dir() - assert "--run-id" not in report.run["reproducibility_command"] + assert f"--run-id {report.run['run_id']}" in report.run["reproducibility_command"] -def test_run_pipeline_mode_sdk_default_run_id_collision_gets_suffix(tmp_path: Path, monkeypatch): +def test_run_pipeline_mode_sdk_default_run_ids_are_unique(tmp_path: Path, monkeypatch): _install_fake_sdk( monkeypatch, best_prompt="optimized prompt", @@ -1326,8 +1345,9 @@ def test_run_pipeline_mode_sdk_default_run_id_collision_gets_suffix(tmp_path: Pa sdk_call_agent="fake_call_agent_module:call_agent", ) - assert first.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z" - assert second.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z-1" + assert first.run["run_id"].startswith("eval_optimize_loop_sdk_") + assert second.run["run_id"].startswith("eval_optimize_loop_sdk_") + assert first.run["run_id"] != second.run["run_id"] assert (tmp_path / "sdk_run" / "runs" / first.run["run_id"]).is_dir() assert (tmp_path / "sdk_run" / "runs" / second.run["run_id"]).is_dir() @@ -1387,12 +1407,14 @@ def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_ decision = report.gate_decisions[0] assert report.selected_candidate is None assert decision.accepted is False - assert decision.gate_status == "partial_applied" + assert decision.gate_status == "applied" + assert decision.not_applied_checks == [] assert decision.validation_score_delta == 0.005 assert any("validation improvement" in reason for reason in decision.reasons) + assert any("cost_unavailable" in reason for reason in decision.reasons) -def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_improvement( +def test_run_pipeline_mode_sdk_custom_gate_uses_full_validation_evaluation( tmp_path: Path, monkeypatch, ): @@ -1401,10 +1423,10 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_impr best_prompt="optimized prompt", baseline_pass_rate=0.5, best_pass_rate=0.75, - pass_rate_improvement=0.25, + pass_rate_improvement=0.9, total_llm_cost=0.123, ) - gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.3, max_total_cost=1.0) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.3, max_total_cost=None) report = run_pipeline( mode="sdk", @@ -1421,6 +1443,7 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_impr assert report.selected_candidate is None assert decision.accepted is False assert decision.validation_score_delta == 0.25 + assert report.audit["sdk_result_summary"]["pass_rate_improvement"] == 0.9 assert any("validation improvement" in reason for reason in decision.reasons) @@ -1449,26 +1472,42 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Pa decision = report.gate_decisions[0] assert report.selected_candidate is None assert decision.accepted is False - assert decision.gate_status == "partial_applied" + assert decision.gate_status == "applied" assert decision.total_run_cost == 2.0 - assert any("cost" in reason for reason in decision.reasons) + assert any("cost_unavailable" in reason for reason in decision.reasons) -def test_sdk_gate_decision_skips_disabled_cost_gate_but_keeps_quality_threshold(): - decision = _sdk_gate_decision( - candidate_id="sdk_best", - sdk_summary={ - "status": "SUCCEEDED", - "pass_rate_improvement": 0.005, - "total_llm_cost": 1000.0, - }, - gate_config={ - "min_val_score_improvement": 0.01, - "max_total_cost": None, - }, +def test_sdk_pipeline_skips_disabled_cost_gate_but_keeps_quality_threshold( + tmp_path: Path, + monkeypatch, +): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.505, + pass_rate_improvement=0.9, + total_llm_cost=1000.0, + ) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=_write_gate_config( + tmp_path, + min_val_score_improvement=0.01, + max_total_cost=None, + ), ) + decision = report.gate_decisions[0] assert decision.accepted is False + assert decision.validation_score_delta == 0.005 assert any("validation improvement" in reason for reason in decision.reasons) assert not any("cost" in reason for reason in decision.reasons) assert decision.total_run_cost == 1000.0 @@ -1672,7 +1711,7 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: "skill_prompt": "optimized skill", } assert "router_prompt" in report.candidates[0]["candidate"].prompt_diff - assert set(report.audit["candidate_prompt_hashes_by_field"]["sdk_best"]) == { + assert set(report.audit["candidate_prompt_hashes"]["sdk_best"]) == { "system_prompt", "router_prompt", "skill_prompt", @@ -1687,12 +1726,15 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: assert (run_dir / "candidate_prompts" / "sdk_best" / "skill_prompt.txt").read_text( encoding="utf-8" ) == "optimized skill" - assert (run_dir / "candidate_prompts" / "sdk_best" / "bundle.txt").read_text( - encoding="utf-8" - ) == report.candidates[0]["candidate"].prompt input_hashes = json.loads((run_dir / "input_hashes.json").read_text(encoding="utf-8")) - assert set(input_hashes["target_prompts"]) == {"system_prompt", "router_prompt", "skill_prompt"} - assert "gate_config" in input_hashes + assert set(input_hashes["target_prompts"]) == { + "system_prompt", + "router_prompt", + "skill_prompt", + } + assert report.audit["gate_config_hash"] + assert report.audit["sdk_result_availability"]["full_train_eval_result"] is True + assert calls["evaluation_count"] == 4 command = report.run["reproducibility_command"] assert "--sdk-call-agent fake_call_agent_module:call_agent" in command assert "--target-prompt" in command @@ -1702,20 +1744,29 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: def test_run_pipeline_mode_sdk_keeps_source_writeback_outside_backend(tmp_path: Path, monkeypatch): calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "system_prompt.txt" + prompt_path.write_bytes(b"baseline prompt\r\n") report = run_pipeline( mode="sdk", train_path=DEFAULT_TRAIN, val_path=DEFAULT_VAL, optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, - prompt_path=DEFAULT_PROMPT, + prompt_path=prompt_path, output_dir=tmp_path / "sdk_run", sdk_call_agent="fake_call_agent_module:call_agent", update_source=True, + gate_config_path=_write_gate_config( + tmp_path, + min_val_score_improvement=0.01, + max_total_cost=None, + ), ) assert report.run["update_source"] is True assert calls["update_source"] is False + assert report.writeback.status == "applied" + assert prompt_path.read_text(encoding="utf-8") == "optimized prompt" assert "--update-source" in report.run["reproducibility_command"] @@ -1798,9 +1849,59 @@ async def optimize(**kwargs): rounds=effective_rounds, ) + class FakeEvalConfig: + def __init__(self, **kwargs): + self.payload = kwargs + + def model_dump_json(self, indent=2): + return json.dumps(self.payload, indent=indent) + + class FakeEvaluationCasesFailed(Exception): + pass + + class FakeAgentEvaluator: + @staticmethod + def get_executer(dataset_path, **kwargs): + evaluation_index = int(calls.get("evaluation_count", 0)) + calls["evaluation_count"] = evaluation_index + 1 + calls.setdefault("evaluation_calls", []).append({ + "dataset_path": dataset_path, + **kwargs, + }) + payload = json.loads(Path(dataset_path).read_text(encoding="utf-8")) + if "eval_cases" in payload: + case_ids = [str(case["eval_id"]) for case in payload["eval_cases"]] + else: + case_ids = [str(case.get("case_id") or case.get("id")) for case in payload["cases"]] + is_baseline = evaluation_index % 4 < 2 + score = baseline_pass_rate if is_baseline else best_pass_rate + result = _sdk_evaluate_result({ + case_id: [ + _sdk_case_run( + case_id, + status="PASSED", + metrics=[_sdk_metric("final_response_avg_score", score, status="PASSED")], + output="evaluated output", + ) + ] + for case_id in case_ids + }) + + class FakeExecuter: + async def evaluate(self): + return None + + def get_result(self): + return result + + return FakeExecuter() + fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") fake_eval_module.AgentOptimizer = FakeAgentOptimizer fake_eval_module.TargetPrompt = FakeTargetPrompt + fake_eval_module.AgentEvaluator = FakeAgentEvaluator + fake_eval_module.EvalConfig = FakeEvalConfig + fake_eval_module.EvaluationCasesFailed = FakeEvaluationCasesFailed monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) call_agent_module = types.ModuleType("fake_call_agent_module") @@ -2031,7 +2132,7 @@ def _write_gate_config( tmp_path: Path, *, min_val_score_improvement: float, - max_total_cost: float, + max_total_cost: float | None, ) -> Path: path = tmp_path / "wrapper_gate.json" path.write_text( From 10dbd96e70576dc706373ceffbdc92df8155017e Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 02:06:36 +0800 Subject: [PATCH 25/34] fix(examples): harden eval optimize audit lifecycle --- .../eval_optimize_loop/eval_loop/artifacts.py | 35 + .../eval_optimize_loop/eval_loop/config.py | 56 +- .../eval_optimize_loop/eval_loop/pipeline.py | 1051 ++++++++++++++--- .../eval_optimize_loop/eval_loop/report.py | 237 ++-- .../eval_optimize_loop/run_pipeline.py | 58 +- .../tests/test_config_validation.py | 148 +++ .../tests/test_pipeline_fake_mode.py | 11 +- .../tests/test_pipeline_orchestration.py | 708 ++++++++++- .../tests/test_sdk_backend.py | 50 +- 9 files changed, 2049 insertions(+), 305 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/eval_loop/artifacts.py diff --git a/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py b/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py new file mode 100644 index 00000000..901e53cc --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py @@ -0,0 +1,35 @@ +"""Cross-platform validation for values used as artifact path components.""" + +from __future__ import annotations + +import re +from typing import Any + + +ARTIFACT_COMPONENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +WINDOWS_RESERVED_NAMES = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{index}" for index in range(1, 10)), + *(f"LPT{index}" for index in range(1, 10)), +} +MAX_ARTIFACT_COMPONENT_LENGTH = 128 + + +def validate_artifact_component(value: Any, *, context: str) -> str: + """Return a portable artifact component or fail before filesystem access.""" + + if ( + not isinstance(value, str) + or value in {"", ".", ".."} + or len(value) > MAX_ARTIFACT_COMPONENT_LENGTH + or value.endswith((".", " ")) + or not ARTIFACT_COMPONENT_RE.fullmatch(value) + ): + raise ValueError(f"unsafe {context}: {value!r}") + windows_stem = value.split(".", 1)[0].upper() + if windows_stem in WINDOWS_RESERVED_NAMES: + raise ValueError(f"unsafe {context}: {value!r} is reserved on Windows") + return value diff --git a/examples/optimization/eval_optimize_loop/eval_loop/config.py b/examples/optimization/eval_optimize_loop/eval_loop/config.py index 66b37b8e..8f092702 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/config.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/config.py @@ -56,9 +56,7 @@ def parse_optimizer_config(payload: dict[str, Any], *, path: str | Path) -> Opti allowed = {"seed", "optimizer", "metrics", "gate"} extras = {key: value for key, value in payload.items() if key not in allowed} - seed = payload.get("seed", 91) - if not isinstance(seed, int): - raise ValueError(f"{path_text}: field 'seed' must be an integer") + seed = resolve_effective_seed(payload, path=path) optimizer = payload.get("optimizer", {}) if not isinstance(optimizer, dict): @@ -82,6 +80,58 @@ def parse_optimizer_config(payload: dict[str, Any], *, path: str | Path) -> Opti ) +def resolve_effective_seed( + payload: dict[str, Any], + *, + path: str | Path, + default: int = 91, + strict_legacy: bool = True, +) -> int: + """Resolve the legacy or official optimizer seed without audit drift. + + The official SDK schema owns ``optimize.algorithm.seed``. A non-integer + top-level ``seed`` may be unrelated SDK metadata and is ignored when the + official nested seed is present. Two integer seed declarations must agree. + """ + + path_text = str(path) + nested_present = False + nested_seed: Any = None + optimize = payload.get("optimize") + if isinstance(optimize, dict): + algorithm = optimize.get("algorithm") + if isinstance(algorithm, dict) and "seed" in algorithm: + nested_present = True + nested_seed = algorithm["seed"] + + top_present = "seed" in payload + top_seed = payload.get("seed") + if nested_present: + nested = _validated_seed( + nested_seed, + field_name=f"{path_text}: field 'optimize.algorithm.seed'", + ) + if top_present and isinstance(top_seed, int) and not isinstance(top_seed, bool): + if top_seed != nested: + raise ValueError( + f"{path_text}: conflicting seed values: top-level seed={top_seed}, " + f"optimize.algorithm.seed={nested}" + ) + return nested + + if not top_present: + return default + if not strict_legacy and (isinstance(top_seed, bool) or not isinstance(top_seed, int)): + return default + return _validated_seed(top_seed, field_name=f"{path_text}: field 'seed'") + + +def _validated_seed(value: Any, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{field_name} must be an integer") + return value + + def validate_inputs( *, train_path: str | Path, diff --git a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py index 2b7878f4..689a3d3e 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py @@ -3,8 +3,11 @@ from __future__ import annotations import hashlib +import json +import math +import os import re -import shlex +import tempfile import time from dataclasses import dataclass from dataclasses import replace @@ -12,27 +15,33 @@ from typing import Any from .attribution import summarize_failures +from .artifacts import validate_artifact_component from .backends import EvaluationBackend from .backends import OptimizationBackend +from .config import _parse_gate_config +from .config import parse_optimizer_config +from .config import resolve_effective_seed from .gate import AcceptanceGate from .loader import read_json -from .loader import sha256_file from .loader import stable_config_hash from .report import build_report from .report import compute_case_deltas from .report import finalize_run_artifacts +from .report import persist_writeback_journal +from .report import persist_writeback_outcome from .report import prepare_run_artifacts from .schemas import CandidatePrompt from .schemas import CostSummary from .schemas import EvalResult from .schemas import OptimizationReport +from .schemas import OptimizationResult from .schemas import WritebackResult from .schemas import to_jsonable from .writeback import commit_prompt_bundle +from .writeback import ConcurrentPromptUpdateError from .writeback import snapshot_prompt_files - -_ARTIFACT_COMPONENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +_REPOSITORY_ROOT = Path(__file__).resolve().parents[4] @dataclass(frozen=True) @@ -51,6 +60,39 @@ class PipelineRequest: run_id: str sdk_call_agent: str | None = None gate_config_path: str | Path | None = None + effective_seed: int | None = None + gate_config_source: str = "request" + + +@dataclass(frozen=True) +class _InputFileSnapshot: + role: str + original_path: Path + snapshot_path: Path + sha256: str + + +@dataclass(frozen=True) +class _InputSnapshots: + files: dict[str, _InputFileSnapshot] + + def path(self, role: str) -> Path: + return self.files[role].snapshot_path + + def hashes(self) -> dict[str, str]: + return {role: item.sha256 for role, item in self.files.items()} + + def cleanup(self) -> None: + failures: list[str] = [] + for item in self.files.values(): + try: + item.snapshot_path.unlink() + except FileNotFoundError: + pass + except OSError as error: + failures.append(f"{item.role}: {error}") + if failures: + raise OSError("failed to remove input snapshots: " + "; ".join(failures)) async def execute_pipeline( @@ -65,57 +107,143 @@ async def execute_pipeline( run_id = _safe_artifact_component(request.run_id, context="run_id") if not request.target_prompt_paths: raise ValueError("target_prompt_paths must not be empty") + _validate_target_prompt_fields(request.target_prompt_paths) + if Path(request.train_path).resolve() == Path(request.validation_path).resolve(): + raise ValueError("train and validation evalset paths must be different") + run_root = _claim_run_directory(request.output_dir, run_id=run_id) + + input_snapshots = _snapshot_pipeline_inputs(request, run_id=run_id) + try: + result = await _execute_with_snapshots( + request, + evaluator=evaluator, + optimizer=optimizer, + started=started, + run_id=run_id, + run_root=run_root, + input_snapshots=input_snapshots, + ) + except BaseException as primary_error: + try: + input_snapshots.cleanup() + except OSError as cleanup_error: + add_note = getattr(primary_error, "add_note", None) + if add_note is not None: + add_note(str(cleanup_error)) + raise + else: + input_snapshots.cleanup() + return result + + +def _validate_target_prompt_fields(target_prompt_paths: dict[str, str | Path]) -> None: + seen: set[str] = set() + resolved_paths: set[Path] = set() + for field_name, prompt_path in target_prompt_paths.items(): + try: + validate_artifact_component(field_name, context="target prompt field") + except ValueError as error: + raise ValueError(f"target prompt field {field_name!r} is unsafe") from error - snapshot = snapshot_prompt_files(request.target_prompt_paths) - baseline_prompts = _decode_snapshot(snapshot) - config_snapshot = read_json(request.optimizer_config_path) + normalized_name = field_name.casefold() + if normalized_name in seen: + raise ValueError("target prompt field names must be case-insensitively unique") + seen.add(normalized_name) + + resolved_path = Path(prompt_path).resolve() + if resolved_path in resolved_paths: + raise ValueError("target prompt fields must not reference the same resolved file") + resolved_paths.add(resolved_path) + + +async def _execute_with_snapshots( + request: PipelineRequest, + *, + evaluator: EvaluationBackend, + optimizer: OptimizationBackend, + started: float, + run_id: str, + run_root: Path, + input_snapshots: _InputSnapshots, +) -> OptimizationReport: + prompt_snapshot = snapshot_prompt_files(request.target_prompt_paths) + baseline_prompts = _decode_snapshot(prompt_snapshot) + raw_config = read_json(input_snapshots.path("optimizer")) + effective_seed = resolve_effective_seed( + raw_config, + path=request.optimizer_config_path, + strict_legacy=request.mode == "fake", + ) + if request.effective_seed is not None and request.effective_seed != effective_seed: + raise ValueError("effective optimizer seed changed between dependency construction and pipeline entry") + if request.mode == "fake": + _validate_backend_seed(evaluator, effective_seed) + if optimizer is not evaluator: + _validate_backend_seed(optimizer, effective_seed) + train_metadata = _strict_case_metadata(input_snapshots.path("train"), role="train") + validation_metadata = _strict_case_metadata( + input_snapshots.path("validation"), + role="validation", + ) + gate_config = _effective_gate_config(request, input_snapshots, raw_config=raw_config) effective_gate_config = _gate_config_with_dataset_protection( - request.gate_config, - validation_path=request.validation_path, + gate_config, + validation_metadata=validation_metadata, ) - run_root = Path(request.output_dir) / "runs" / run_id + train_path = input_snapshots.path("train") + validation_path = input_snapshots.path("validation") + optimizer_path = input_snapshots.path("optimizer") + _verify_snapshot_integrity(input_snapshots) + _verify_snapshot_integrity(input_snapshots, "train") baseline_train = await evaluator.evaluate( prompt_id="baseline", prompts=baseline_prompts, - dataset_path=request.train_path, + dataset_path=train_path, split="train", trace=request.trace, artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "train"), ) - _validate_result_identity( + _verify_snapshot_integrity(input_snapshots, "train") + _validate_eval_result( baseline_train, expected_prompt_id="baseline", expected_split="train", + expected_case_ids=set(train_metadata), ) + _verify_snapshot_integrity(input_snapshots, "validation") baseline_validation = await evaluator.evaluate( prompt_id="baseline", prompts=baseline_prompts, - dataset_path=request.validation_path, + dataset_path=validation_path, split="validation", trace=request.trace, artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "validation"), ) - _validate_result_identity( + _verify_snapshot_integrity(input_snapshots, "validation") + _validate_eval_result( baseline_validation, expected_prompt_id="baseline", expected_split="validation", + expected_case_ids=set(validation_metadata), ) failure_summary = summarize_failures([baseline_train]) - optimizer_artifact_dir = _artifact_dir(run_root, "optimizer") + _verify_snapshot_integrity(input_snapshots, "train", "validation", "optimizer") optimization = await optimizer.optimize_candidates( baseline_prompts=baseline_prompts, baseline_train=baseline_train, failure_summary=failure_summary, - train_path=request.train_path, - validation_path=request.validation_path, - config_path=request.optimizer_config_path, - artifact_dir=optimizer_artifact_dir, + train_path=train_path, + validation_path=validation_path, + config_path=optimizer_path, + artifact_dir=_artifact_dir(run_root, "optimizer"), ) + _verify_snapshot_integrity(input_snapshots, "train", "validation", "optimizer") + _validate_optimization_result(optimization) candidate_bundles = _validated_candidate_bundles( optimization.candidates, - expected_fields=set(snapshot.files), + expected_fields=set(prompt_snapshot.files), ) gate = AcceptanceGate(effective_gate_config) @@ -129,29 +257,33 @@ async def execute_pipeline( for index, candidate in enumerate(optimization.candidates, start=1): bundle = candidate_bundles[candidate.candidate_id] - path_label = f"{index:03d}_{_path_label(candidate.candidate_id)}" + path_label = _candidate_artifact_component(index, candidate.candidate_id) + _verify_snapshot_integrity(input_snapshots, "train") train_result = await evaluator.evaluate( prompt_id=candidate.candidate_id, prompts=bundle, - dataset_path=request.train_path, + dataset_path=train_path, split="train", trace=request.trace, artifact_dir=_artifact_dir(run_root, "evaluations", path_label, "train"), ) - _validate_result_identity( + _verify_snapshot_integrity(input_snapshots, "train") + _validate_eval_result( train_result, expected_prompt_id=candidate.candidate_id, expected_split="train", ) + _verify_snapshot_integrity(input_snapshots, "validation") validation_result = await evaluator.evaluate( prompt_id=candidate.candidate_id, prompts=bundle, - dataset_path=request.validation_path, + dataset_path=validation_path, split="validation", trace=request.trace, artifact_dir=_artifact_dir(run_root, "evaluations", path_label, "validation"), ) - _validate_result_identity( + _verify_snapshot_integrity(input_snapshots, "validation") + _validate_eval_result( validation_result, expected_prompt_id=candidate.candidate_id, expected_split="validation", @@ -181,15 +313,19 @@ async def execute_pipeline( cost_summary=optimization.cost, cumulative_cost=cumulative_cost, ) - candidate_records.append({ - "candidate": candidate, - "train_result": train_result, - "validation_result": validation_result, - }) + candidate_records.append( + { + "candidate": candidate, + "train_result": train_result, + "validation_result": validation_result, + } + ) all_deltas.extend(deltas) gate_decisions.append(decision) - candidate_eval_cost = round(train_result.cost + validation_result.cost, 6) - explicit_evaluator_cost = round(explicit_evaluator_cost + candidate_eval_cost, 6) + explicit_evaluator_cost = round( + explicit_evaluator_cost + train_result.cost + validation_result.cost, + 6, + ) cumulative_cost = decision.total_run_cost selected_candidate = _select_candidate(candidate_records, gate_decisions) @@ -200,26 +336,30 @@ async def execute_pipeline( total=cumulative_cost, complete=optimization.cost.complete, ) + _validate_cost_summary(cost_summary, context="pipeline cost summary") input_hashes: dict[str, Any] = { - "train": sha256_file(request.train_path), - "validation": sha256_file(request.validation_path), - "optimizer": sha256_file(request.optimizer_config_path), - "target_prompts": snapshot.hashes(), + **input_snapshots.hashes(), + "target_prompts": prompt_snapshot.hashes(), } - if request.gate_config_path is not None: - input_hashes["gate_config"] = sha256_file(request.gate_config_path) - before_hashes = snapshot.hashes() + before_hashes = prompt_snapshot.hashes() writeback_journal = _initial_writeback_journal( + run_id=run_id, selected_candidate=selected_candidate, update_source=request.update_source, before_hashes=before_hashes, + candidate_hashes=( + _prompt_bundle_hashes(candidate_bundles[selected_candidate]) if selected_candidate is not None else {} + ), + input_hashes=input_snapshots.hashes(), ) audit = _build_audit( request=request, started=started, - snapshot_hashes=snapshot.hashes(), + snapshot_hashes=prompt_snapshot.hashes(), input_hashes=input_hashes, - config_snapshot=config_snapshot, + config_snapshot=_sanitize_public_value(raw_config), + raw_config_hash=stable_config_hash(raw_config), + effective_seed=effective_seed, baseline_train=baseline_train, baseline_validation=baseline_validation, candidate_records=candidate_records, @@ -231,16 +371,9 @@ async def execute_pipeline( effective_gate_config=effective_gate_config, writeback_journal=writeback_journal, ) - run = _build_run( - request, - baseline_train=baseline_train, - baseline_validation=baseline_validation, - ) + run = _build_run(request, baseline_train=baseline_train, baseline_validation=baseline_validation) if selected_candidate is None: - provisional_writeback = WritebackResult( - status="rejected", - before_hashes=before_hashes, - ) + provisional_writeback = WritebackResult(status="rejected", before_hashes=before_hashes) else: provisional_writeback = WritebackResult( status="not_requested", @@ -266,37 +399,384 @@ async def execute_pipeline( ) artifact_paths = prepare_run_artifacts(report, request.output_dir) - if selected_candidate is None: - writeback = provisional_writeback - elif not request.update_source: - writeback = provisional_writeback - else: + if selected_candidate is None or not request.update_source: + terminal_journal = _terminal_journal( + writeback_journal, + provisional_writeback, + state=provisional_writeback.status, + ) + final_report = _with_writeback(report, terminal_journal, provisional_writeback) + persist_writeback_outcome(final_report, artifact_paths) + finalize_run_artifacts(final_report, artifact_paths) + return final_report + + input_drift = _public_input_drift(_input_drift(input_snapshots), request) + if input_drift: + error = "input changed after pipeline entry: " + ", ".join(sorted(input_drift)) + writeback = WritebackResult( + status="rejected", + before_hashes=before_hashes, + after_hashes=_current_prompt_hashes(prompt_snapshot), + error=error, + ) + final_journal = _terminal_journal( + writeback_journal, + writeback, + state="conflict", + input_drift=input_drift, + ) + final_report = _with_writeback(report, final_journal, writeback, input_drift=input_drift) + persist_writeback_outcome(final_report, artifact_paths) + finalize_run_artifacts(final_report, artifact_paths) + return final_report + + committing_journal = dict(writeback_journal) + committing_journal.update({"state": "committing", "report_phase": "writeback"}) + persist_writeback_journal(artifact_paths, committing_journal) + try: writeback = commit_prompt_bundle( - snapshot, + prompt_snapshot, candidate_bundles[selected_candidate], ) - final_audit = dict(report.audit) - final_journal = dict(writeback_journal) - final_journal.update({ - "state": writeback.status, - "after_hashes": dict(writeback.after_hashes), - "error": writeback.error, - }) - final_audit["writeback_journal"] = final_journal - final_report = replace(report, audit=final_audit, writeback=writeback) + except ConcurrentPromptUpdateError as error: + writeback = WritebackResult( + status="rejected", + before_hashes=before_hashes, + after_hashes=_current_prompt_hashes(prompt_snapshot), + error=_sanitize_error_message( + f"prompt writeback conflict: {error}", + request, + ), + ) + terminal_state = "conflict" + else: + writeback = _sanitize_writeback_result(writeback, request) + terminal_state = writeback.status + final_journal = _terminal_journal( + committing_journal, + writeback, + state=terminal_state, + ) + final_report = _with_writeback(report, final_journal, writeback) + try: + persist_writeback_journal(artifact_paths, final_journal) + except Exception as persist_error: + unknown_journal = dict(final_journal) + unknown_journal.update( + { + "state": "unknown", + "error": _sanitize_error_message( + f"failed to persist terminal writeback outcome: {persist_error}", + request, + ), + "observed_hashes": _current_prompt_hashes(prompt_snapshot), + } + ) + try: + persist_writeback_journal(artifact_paths, unknown_journal) + except Exception: + pass + raise + persist_writeback_outcome(final_report, artifact_paths) finalize_run_artifacts(final_report, artifact_paths) return final_report +def _snapshot_pipeline_inputs(request: PipelineRequest, *, run_id: str) -> _InputSnapshots: + sources: dict[str, Path] = { + "train": Path(request.train_path), + "validation": Path(request.validation_path), + "optimizer": Path(request.optimizer_config_path), + } + if request.gate_config_path is not None: + sources["gate_config"] = Path(request.gate_config_path) + snapshots: dict[str, _InputFileSnapshot] = {} + try: + for role, source in sources.items(): + content = source.read_bytes() + fd, temp_name = tempfile.mkstemp( + dir=source.parent, + prefix=f".{run_id}.{role}.", + suffix=f".snapshot-{source.name}", + ) + snapshot_path = Path(temp_name) + try: + with os.fdopen(fd, "wb") as stream: + fd = -1 + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + except BaseException as primary_error: + try: + snapshot_path.unlink() + except FileNotFoundError: + pass + except OSError as cleanup_error: + add_note = getattr(primary_error, "add_note", None) + if add_note is not None: + add_note(f"failed to remove partial input snapshot: {cleanup_error}") + raise + finally: + if fd >= 0: + os.close(fd) + snapshots[role] = _InputFileSnapshot( + role=role, + original_path=source, + snapshot_path=snapshot_path, + sha256=hashlib.sha256(content).hexdigest(), + ) + except BaseException as primary_error: + try: + _InputSnapshots(snapshots).cleanup() + except OSError as cleanup_error: + add_note = getattr(primary_error, "add_note", None) + if add_note is not None: + add_note(str(cleanup_error)) + raise + return _InputSnapshots(snapshots) + + +def _claim_run_directory(output_dir: str | Path, *, run_id: str) -> Path: + runs_root = Path(output_dir) / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + run_root = runs_root / run_id + try: + run_root.mkdir() + except FileExistsError as error: + raise ValueError(f"run_id {run_id!r} already exists in {runs_root}") from error + return run_root + + +def _verify_snapshot_integrity( + snapshots: _InputSnapshots, + *requested_roles: str, +) -> None: + roles = requested_roles or tuple(snapshots.files) + for role in roles: + item = snapshots.files[role] + try: + observed = hashlib.sha256(item.snapshot_path.read_bytes()).hexdigest() + except OSError as error: + raise ValueError(f"immutable input snapshot {role!r} is unavailable: {error}") from error + if observed != item.sha256: + raise ValueError(f"immutable input snapshot changed for {role}") + + +def _effective_gate_config( + request: PipelineRequest, + snapshots: _InputSnapshots, + *, + raw_config: dict[str, Any], +) -> dict[str, Any]: + if "gate_config" in snapshots.files: + payload = read_json(snapshots.path("gate_config")) + gate_payload = payload.get("gate", payload) + if gate_payload is None: + gate_payload = {} + if not isinstance(gate_payload, dict): + raise ValueError("gate config field 'gate' must be an object") + elif request.gate_config_source == "optimizer": + gate_payload = parse_optimizer_config( + raw_config, + path=request.optimizer_config_path, + ).gate.to_dict() + elif request.gate_config_source == "request": + gate_payload = request.gate_config + else: + raise ValueError(f"unknown gate_config_source: {request.gate_config_source!r}") + if not isinstance(gate_payload, dict): + raise ValueError("gate config must be an object") + return _parse_gate_config(gate_payload, path="gate config snapshot").to_dict() + + +def _validate_backend_seed(backend: Any, effective_seed: int) -> None: + if not hasattr(backend, "seed"): + return + backend_seed = getattr(backend, "seed") + if ( + isinstance(backend_seed, bool) + or not isinstance(backend_seed, int) + or backend_seed != effective_seed + ): + raise ValueError( + f"fake backend seed {backend_seed!r} does not match effective seed {effective_seed}" + ) + + +def _strict_case_metadata(path: str | Path, *, role: str) -> dict[str, bool]: + payload = read_json(path) + has_standard = "eval_cases" in payload + has_legacy = "cases" in payload + if has_standard == has_legacy: + raise ValueError( + f"{role} evalset must contain exactly one of eval_cases or cases" + ) + if has_standard: + cases = payload["eval_cases"] + standard = True + else: + cases = payload["cases"] + standard = False + if not isinstance(cases, list): + raise ValueError(f"{role} evalset cases must be a list") + if not cases: + raise ValueError(f"{role} evalset cases must not be empty") + + metadata: dict[str, bool] = {} + for index, case in enumerate(cases): + if not isinstance(case, dict): + raise ValueError(f"{role} evalset case {index} must be an object") + if standard: + case_id = case.get("eval_id") + protected = False + if "session_input" in case: + session_input = case["session_input"] + if not isinstance(session_input, dict): + raise ValueError(f"{role} evalset case {index} session_input must be an object") + state = session_input.get("state", {}) + if state is None: + state = {} + if not isinstance(state, dict): + raise ValueError(f"{role} evalset case {index} session_input.state must be an object") + protected = state.get("eval_optimize_protected", False) + else: + case_id = case["case_id"] if "case_id" in case else case.get("id") + protected = case.get("protected", False) + + if not isinstance(case_id, str) or not case_id.strip(): + raise ValueError(f"{role} evalset case {index} case ID must be a non-empty string") + if not isinstance(protected, bool): + raise ValueError(f"{role} evalset case {case_id!r} protected metadata must be a boolean") + if case_id in metadata: + raise ValueError(f"{role} evalset contains duplicate case ID {case_id!r}") + metadata[case_id] = protected + return metadata + + +def _input_drift(snapshots: _InputSnapshots) -> dict[str, dict[str, Any]]: + drift: dict[str, dict[str, Any]] = {} + for role, item in snapshots.files.items(): + try: + after_hash = hashlib.sha256(item.original_path.read_bytes()).hexdigest() + except OSError as error: + drift[role] = { + "before_hash": item.sha256, + "after_hash": None, + "error": str(error), + } + continue + if after_hash != item.sha256: + drift[role] = { + "before_hash": item.sha256, + "after_hash": after_hash, + "error": None, + } + return drift + + +def _public_input_drift( + drift: dict[str, dict[str, Any]], + request: PipelineRequest, +) -> dict[str, dict[str, Any]]: + public: dict[str, dict[str, Any]] = {} + for role, details in drift.items(): + sanitized = dict(details) + if sanitized.get("error"): + sanitized["error"] = _sanitize_error_message(str(sanitized["error"]), request) + public[role] = sanitized + return public + + +def _prompt_bundle_hashes(bundle: dict[str, str]) -> dict[str, str]: + return {name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() for name, prompt in bundle.items()} + + +def _current_prompt_hashes(snapshot: Any) -> dict[str, str]: + hashes: dict[str, str] = {} + for name, prompt_file in snapshot.files.items(): + try: + hashes[name] = hashlib.sha256(prompt_file.path.read_bytes()).hexdigest() + except OSError: + continue + return hashes + + +def _terminal_journal( + journal: dict[str, Any], + writeback: WritebackResult, + *, + state: str, + input_drift: dict[str, Any] | None = None, +) -> dict[str, Any]: + terminal = dict(journal) + terminal.update( + { + "state": state, + "report_phase": "final", + "after_hashes": dict(writeback.after_hashes), + "error": writeback.error, + } + ) + if input_drift: + terminal["input_drift"] = input_drift + return terminal + + +def _with_writeback( + report: OptimizationReport, + journal: dict[str, Any], + writeback: WritebackResult, + *, + input_drift: dict[str, Any] | None = None, +) -> OptimizationReport: + audit = dict(report.audit) + audit["writeback_journal"] = journal + if input_drift: + audit["input_drift"] = input_drift + return replace(report, audit=audit, writeback=writeback) + + +def _sanitize_writeback_result( + writeback: WritebackResult, + request: PipelineRequest, +) -> WritebackResult: + if writeback.error is None: + return writeback + return replace( + writeback, + error=_sanitize_error_message(writeback.error, request), + ) + + +def _sanitize_error_message(message: str, request: PipelineRequest) -> str: + replacements: list[tuple[str, str]] = [] + raw_paths: list[str | Path] = [ + request.train_path, + request.validation_path, + request.optimizer_config_path, + request.output_dir, + *request.target_prompt_paths.values(), + ] + if request.gate_config_path is not None: + raw_paths.append(request.gate_config_path) + for raw_path in raw_paths: + path = Path(raw_path) + display = _display_path(path) + replacements.append((str(path), display)) + replacements.append((str(path.resolve()), display)) + sanitized = message + for source, replacement in sorted(replacements, key=lambda item: len(item[0]), reverse=True): + sanitized = sanitized.replace(source, replacement) + return sanitized + + def _decode_snapshot(snapshot: Any) -> dict[str, str]: prompts: dict[str, str] = {} for name, prompt_file in snapshot.files.items(): try: prompts[name] = prompt_file.content.decode("utf-8", errors="strict") except UnicodeDecodeError as exc: - raise ValueError( - f"target prompt {name!r} at {prompt_file.path} is not valid UTF-8" - ) from exc + raise ValueError(f"target prompt {name!r} at {prompt_file.path} is not valid UTF-8") from exc return prompts @@ -306,14 +786,26 @@ def _validated_candidate_bundles( expected_fields: set[str], ) -> dict[str, dict[str, str]]: bundles: dict[str, dict[str, str]] = {} + casefold_ids: set[str] = set() for candidate in candidates: candidate_id = candidate.candidate_id - if not isinstance(candidate_id, str) or not candidate_id: - raise ValueError("candidate_id must be a non-empty string") - if candidate_id == "baseline": + _validate_candidate_id(candidate_id) + for field_name, value in ( + ("prompt", candidate.prompt), + ("rationale", candidate.rationale), + ("prompt_diff", candidate.prompt_diff), + ): + _validate_utf8_text( + value, + context=f"candidate {candidate_id!r} {field_name}", + ) + if candidate_id.casefold() == "baseline": raise ValueError("candidate_id 'baseline' is reserved") if candidate_id in bundles: raise ValueError(f"duplicate candidate_id: {candidate_id}") + if candidate_id.casefold() in casefold_ids: + raise ValueError(f"candidate_id values must be case-insensitively unique: {candidate_id}") + casefold_ids.add(candidate_id.casefold()) bundle = candidate.bundle() actual_fields = set(bundle) if actual_fields != expected_fields: @@ -325,80 +817,235 @@ def _validated_candidate_bundles( for field_name, prompt_text in bundle.items(): if not isinstance(prompt_text, str) or not prompt_text: raise ValueError( - f"candidate {candidate_id!r} bundle field {field_name!r} " - "must be a non-empty string" + f"candidate {candidate_id!r} bundle field {field_name!r} " "must be a non-empty string" ) + _validate_utf8_text( + prompt_text, + context=f"candidate {candidate_id!r} bundle field {field_name!r}", + ) bundles[candidate_id] = dict(bundle) return bundles +def _validate_candidate_id(candidate_id: Any) -> None: + try: + validate_artifact_component(candidate_id, context="candidate_id") + except ValueError as error: + raise ValueError(f"candidate_id is not artifact-safe: {candidate_id!r}") from error + + +def _validate_utf8_text(value: Any, *, context: str) -> None: + if not isinstance(value, str): + raise ValueError(f"{context} must be a string") + try: + value.encode("utf-8", errors="strict") + except UnicodeEncodeError as error: + raise ValueError(f"{context} must be valid UTF-8") from error + + def _result_pair_is_comparable(baseline: EvalResult, candidate: EvalResult) -> bool: baseline_ids = [case.case_id for case in baseline.cases] candidate_ids = [case.case_id for case in candidate.cases] - return bool(baseline_ids) and bool(candidate_ids) and ( - len(baseline_ids) == len(set(baseline_ids)) - and len(candidate_ids) == len(set(candidate_ids)) - and set(baseline_ids) == set(candidate_ids) + return ( + bool(baseline_ids) + and bool(candidate_ids) + and ( + len(baseline_ids) == len(set(baseline_ids)) + and len(candidate_ids) == len(set(candidate_ids)) + and set(baseline_ids) == set(candidate_ids) + ) ) -def _validate_result_identity( +def _validate_eval_result( result: EvalResult, *, expected_prompt_id: str, expected_split: str, + expected_case_ids: set[str] | None = None, ) -> None: if result.prompt_id != expected_prompt_id: raise ValueError( - f"evaluation result prompt_id mismatch: expected {expected_prompt_id!r}, " - f"got {result.prompt_id!r}" + f"evaluation result prompt_id mismatch: expected {expected_prompt_id!r}, " f"got {result.prompt_id!r}" ) if result.split != expected_split: raise ValueError( f"EvalResult split mismatch for {expected_prompt_id!r}: " f"expected {expected_split!r}, got {result.split!r}" ) + score = _finite_number( + result.score, + context=f"EvalResult score for {expected_prompt_id!r}", + minimum=0.0, + maximum=1.0, + ) + _finite_number( + result.cost, + context=f"EvalResult cost for {expected_prompt_id!r}", + minimum=0.0, + ) + if not isinstance(result.passed, bool): + raise ValueError(f"EvalResult passed for {expected_prompt_id!r} must be a boolean") + case_ids: set[str] = set() for case in result.cases: + if not isinstance(case.case_id, str) or not case.case_id: + raise ValueError("CaseResult case_id must be a non-empty string") + if case.case_id in case_ids: + # Duplicate IDs are handled by the fail-closed gate, but the numeric + # protocol boundary still validates each record independently. + pass + case_ids.add(case.case_id) if case.split != expected_split: raise ValueError( f"CaseResult {case.case_id!r} split mismatch for {expected_prompt_id!r}: " f"expected {expected_split!r}, got {case.split!r}" ) + _finite_number( + case.score, + context=f"CaseResult {case.case_id!r} score", + minimum=0.0, + maximum=1.0, + ) + _finite_number( + case.cost, + context=f"CaseResult {case.case_id!r} cost", + minimum=0.0, + ) + if not isinstance(case.passed, bool) or not isinstance(case.hard_failed, bool): + raise ValueError(f"CaseResult {case.case_id!r} passed/hard_failed must be booleans") + if not isinstance(case.metrics, dict): + raise ValueError(f"CaseResult {case.case_id!r} metrics must be an object") + for metric_name, metric_value in case.metrics.items(): + if not isinstance(metric_name, str) or not metric_name: + raise ValueError(f"CaseResult {case.case_id!r} metric names must be strings") + _finite_number( + metric_value, + context=f"CaseResult {case.case_id!r} metric {metric_name!r}", + ) + if result.cases: + mean_score = sum(float(case.score) for case in result.cases) / len(result.cases) + if not math.isclose(score, mean_score, rel_tol=0.0, abs_tol=1e-6): + raise ValueError(f"EvalResult score for {expected_prompt_id!r} must equal the case mean") + expected_passed = bool(result.cases) and all(case.passed for case in result.cases) + if result.passed != expected_passed: + raise ValueError(f"EvalResult passed for {expected_prompt_id!r} must agree with case results") + if expected_case_ids is not None: + observed_ids = [case.case_id for case in result.cases] + if len(observed_ids) != len(set(observed_ids)) or set(observed_ids) != expected_case_ids: + raise ValueError( + f"{expected_prompt_id} {expected_split} result must exactly match dataset case IDs; " + f"expected={sorted(expected_case_ids)}, observed={sorted(set(observed_ids))}" + ) + + +def _validate_optimization_result(result: OptimizationResult) -> None: + if not isinstance(result.candidates, list) or not isinstance(result.rounds, list): + raise ValueError("optimization result candidates and rounds must be lists") + _validate_cost_summary(result.cost, context="optimization cost") + for index, round_record in enumerate(result.rounds): + if ( + isinstance(round_record.round_id, bool) + or not isinstance(round_record.round_id, int) + or round_record.round_id <= 0 + ): + raise ValueError(f"optimization round {index} round_id must be a positive integer") + _validate_candidate_id(round_record.candidate_id) + _finite_number( + round_record.duration_seconds, + context=f"optimization round {round_record.round_id} duration", + minimum=0.0, + ) + if not isinstance(round_record.metrics, dict): + raise ValueError(f"optimization round {round_record.round_id} metrics must be an object") + for metric_name, metric_value in round_record.metrics.items(): + _finite_number( + metric_value, + context=f"optimization round {round_record.round_id} metric {metric_name!r}", + ) + _validate_cost_summary( + round_record.cost, + context=f"optimization round {round_record.round_id} cost", + ) + if not isinstance(result.raw_summary, dict): + raise ValueError("optimization raw_summary must be an object") + _validate_nested_numbers(result.raw_summary, context="optimization raw_summary") + + +def _validate_cost_summary(summary: CostSummary, *, context: str) -> None: + components = { + "optimizer": _finite_number(summary.optimizer, context=f"{context} optimizer", minimum=0.0), + "evaluator": _finite_number(summary.evaluator, context=f"{context} evaluator", minimum=0.0), + "agent": _finite_number(summary.agent, context=f"{context} agent", minimum=0.0), + } + total = _finite_number(summary.total, context=f"{context} total", minimum=0.0) + if not isinstance(summary.complete, bool): + raise ValueError(f"{context} complete must be a boolean") + if summary.complete and not math.isclose( + total, + sum(components.values()), + rel_tol=0.0, + abs_tol=1e-6, + ): + raise ValueError(f"{context} total must equal components when complete") + + +def _validate_nested_numbers(value: Any, *, context: str) -> None: + if isinstance(value, bool) or value is None or isinstance(value, str): + return + if isinstance(value, (int, float)): + _finite_number(value, context=context) + return + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"{context} keys must be JSON-compatible strings") + _validate_nested_numbers(item, context=f"{context}.{key}") + return + if isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _validate_nested_numbers(item, context=f"{context}[{index}]") + return + raise ValueError(f"{context} contains non JSON-compatible value {type(value).__name__}") + + +def _finite_number( + value: Any, + *, + context: str, + minimum: float | None = None, + maximum: float | None = None, +) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{context} must be a finite number") + try: + number = float(value) + except OverflowError as error: + raise ValueError(f"{context} must be a finite number") from error + if not math.isfinite(number): + raise ValueError(f"{context} must be a finite number") + if minimum is not None and number < minimum: + raise ValueError(f"{context} must be non-negative" if minimum == 0 else f"{context} is too small") + if maximum is not None and number > maximum: + raise ValueError(f"{context} must be between {minimum:g} and {maximum:g}") + return number def _gate_config_with_dataset_protection( gate_config: dict[str, Any], *, - validation_path: str | Path, + validation_metadata: dict[str, bool], ) -> dict[str, Any]: effective = dict(gate_config) configured_ids = effective.get("protected_case_ids", []) if not isinstance(configured_ids, list) or not all( - isinstance(case_id, str) for case_id in configured_ids + isinstance(case_id, str) and bool(case_id.strip()) for case_id in configured_ids ): - raise ValueError("gate protected_case_ids must be a list of strings") + raise ValueError("gate protected_case_ids must be a list of non-empty strings") + missing_ids = sorted(set(configured_ids) - set(validation_metadata)) + if missing_ids: + raise ValueError("gate protected_case_ids references missing validation cases: " + ", ".join(missing_ids)) protected_ids = set(configured_ids) - payload = read_json(validation_path) - standard_cases = payload.get("eval_cases") - legacy_cases = payload.get("cases") - if isinstance(standard_cases, list): - for case in standard_cases: - if not isinstance(case, dict): - continue - session_input = case.get("session_input") - state = session_input.get("state") if isinstance(session_input, dict) else None - protected = state.get("eval_optimize_protected", False) if isinstance(state, dict) else False - if protected is True: - case_id = case.get("eval_id") - if isinstance(case_id, str) and case_id: - protected_ids.add(case_id) - elif isinstance(legacy_cases, list): - for case in legacy_cases: - if not isinstance(case, dict) or case.get("protected") is not True: - continue - case_id = case.get("case_id") or case.get("id") - if isinstance(case_id, str) and case_id: - protected_ids.add(case_id) + protected_ids.update(case_id for case_id, protected in validation_metadata.items() if protected) effective["protected_case_ids"] = sorted(protected_ids) return effective @@ -432,7 +1079,7 @@ def _build_run( baseline_train: EvalResult, baseline_validation: EvalResult, ) -> dict[str, Any]: - target_paths = {name: str(path) for name, path in request.target_prompt_paths.items()} + target_paths = {name: _display_path(path) for name, path in request.target_prompt_paths.items()} return { "run_id": request.run_id, "mode": request.mode, @@ -444,14 +1091,13 @@ def _build_run( "target_prompt_paths": target_paths, "prompt_source": target_paths.get("system_prompt"), "paths": { - "train": str(request.train_path), - "validation": str(request.validation_path), - "optimizer": str(request.optimizer_config_path), + "train": _display_path(request.train_path), + "validation": _display_path(request.validation_path), + "optimizer": _display_path(request.optimizer_config_path), "prompts": target_paths, }, - "reproducibility_command": ( - _reproducibility_command(request) - ), + "reproducibility_shell": "powershell", + "reproducibility_command": _reproducibility_command(request), } @@ -462,6 +1108,8 @@ def _build_audit( snapshot_hashes: dict[str, str], input_hashes: dict[str, Any], config_snapshot: dict[str, Any], + raw_config_hash: str, + effective_seed: int, baseline_train: EvalResult, baseline_validation: EvalResult, candidate_records: list[dict[str, Any]], @@ -481,37 +1129,43 @@ def _build_audit( for record in candidate_records } candidate_prompt_hashes = { - candidate_id: { - name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() - for name, prompt in bundle.items() - } + candidate_id: {name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() for name, prompt in bundle.items()} for candidate_id, bundle in candidate_bundles.items() } duration = max(time.perf_counter() - started, 1e-9) - target_paths = {name: str(path) for name, path in request.target_prompt_paths.items()} + target_paths = {name: _display_path(path) for name, path in request.target_prompt_paths.items()} return { - "seed": _config_seed(config_snapshot), + "seed": effective_seed, "duration_seconds": duration, - "config_hash": stable_config_hash(config_snapshot), + "config_hash": raw_config_hash, + "config_file_sha256": input_hashes["optimizer"], + "redacted_config_hash": stable_config_hash(config_snapshot), + "redacted_config_snapshot_sha256": _pretty_json_sha256(config_snapshot), "config_snapshot": config_snapshot, "gate_config_hash": stable_config_hash(effective_gate_config), "gate_config_snapshot": effective_gate_config, "writeback_journal": writeback_journal, "input_hashes": input_hashes, "input_paths": { - "train": str(request.train_path), - "validation": str(request.validation_path), - "optimizer": str(request.optimizer_config_path), + "train": _display_path(request.train_path), + "validation": _display_path(request.validation_path), + "optimizer": _display_path(request.optimizer_config_path), "prompts": target_paths, **({"prompt": target_paths["system_prompt"]} if "system_prompt" in target_paths else {}), }, "prompt_hash": snapshot_hashes.get("system_prompt"), "prompt_hashes": snapshot_hashes, "candidate_prompt_hashes": candidate_prompt_hashes, + "candidate_artifacts": { + record["candidate"].candidate_id: _candidate_artifact_component( + index, + record["candidate"].candidate_id, + ) + for index, record in enumerate(candidate_records, start=1) + }, "candidate_prompts": candidate_bundles, "prompt_diffs": { - record["candidate"].candidate_id: record["candidate"].prompt_diff - for record in candidate_records + record["candidate"].candidate_id: record["candidate"].prompt_diff for record in candidate_records }, "total_run_cost": cost_summary.total, "cost": { @@ -522,17 +1176,14 @@ def _build_audit( "total": cost_summary.total, "complete": cost_summary.complete, }, - "sdk_result_summary": to_jsonable(optimization_raw_summary), + "sdk_result_summary": _sanitize_public_value(to_jsonable(optimization_raw_summary)), "sdk_result_availability": { - "aggregate_validation_result": bool( - request.mode == "sdk" and optimization_raw_summary - ), + "aggregate_validation_result": bool(request.mode == "sdk" and optimization_raw_summary), "full_train_eval_result": True, "full_per_case_validation_delta": all_results_comparable, }, - "reproducibility_command": ( - _reproducibility_command(request) - ), + "reproducibility_shell": "powershell", + "reproducibility_command": _reproducibility_command(request), } @@ -543,27 +1194,102 @@ def _artifact_dir(root: Path, *parts: str) -> Path: def _safe_artifact_component(value: str, *, context: str) -> str: - if value in {"", ".", ".."} or not _ARTIFACT_COMPONENT_RE.fullmatch(value): - raise ValueError(f"unsafe {context}: {value!r}") + return validate_artifact_component(value, context=context) + + +def _candidate_artifact_component(index: int, candidate_id: str) -> str: + digest = hashlib.sha256(candidate_id.encode("utf-8")).hexdigest()[:12] + return f"{index:03d}-{digest}" + + +def _display_path(raw_path: str | Path) -> str: + path = Path(raw_path).resolve() + try: + relative = path.relative_to(_REPOSITORY_ROOT) + except ValueError: + return f"$EXTERNAL/{path.name}" + return relative.as_posix() + + +def _redact_secrets(value: Any) -> Any: + if isinstance(value, dict): + redacted: dict[str, Any] = {} + for key, item in value.items(): + key_text = str(key) + lowered = key_text.casefold() + if _is_secret_key(lowered): + redacted[key_text] = "" + else: + redacted[key_text] = _redact_secrets(item) + return redacted + if isinstance(value, list): + return [_redact_secrets(item) for item in value] + if isinstance(value, tuple): + return [_redact_secrets(item) for item in value] return value -def _path_label(value: str) -> str: - label = re.sub(r"[^A-Za-z0-9_.-]", "_", value) - if label in {"", ".", ".."}: - return "candidate" - return label +def _pretty_json_sha256(value: Any) -> str: + content = json.dumps( + value, + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + "\n" + return hashlib.sha256(content.encode("utf-8")).hexdigest() -def _config_seed(config_snapshot: dict[str, Any]) -> Any: - if "seed" in config_snapshot: - return config_snapshot["seed"] - optimize = config_snapshot.get("optimize") - if isinstance(optimize, dict): - algorithm = optimize.get("algorithm") - if isinstance(algorithm, dict) and "seed" in algorithm: - return algorithm["seed"] - return None +def _is_secret_key(lowered: str) -> bool: + normalized = lowered.replace("-", "_").replace(" ", "_") + if normalized in { + "api_key", + "apikey", + "token", + "access_token", + "refresh_token", + "auth_token", + "password", + "passwd", + "credentials", + "credential", + "secret", + "authorization", + "private_key", + "signing_key", + "ssh_key", + }: + return True + if normalized.endswith("_token"): + return True + return any( + marker in normalized + for marker in ( + "api_key", + "password", + "passwd", + "credential", + "secret", + "authorization", + "private_key", + ) + ) + + +def _sanitize_public_value(value: Any) -> Any: + value = _redact_secrets(value) + if isinstance(value, dict): + return {str(key): _sanitize_public_value(item) for key, item in value.items()} + if isinstance(value, list): + return [_sanitize_public_value(item) for item in value] + if isinstance(value, str): + try: + candidate = Path(value) + if candidate.is_absolute(): + return _display_path(candidate) + except (OSError, ValueError): + pass + return value def _reproducibility_command(request: PipelineRequest) -> str: @@ -573,38 +1299,51 @@ def _reproducibility_command(request: PipelineRequest) -> str: "--mode", request.mode, "--train", - str(request.train_path), + _display_path(request.train_path), "--val", - str(request.validation_path), + _display_path(request.validation_path), "--optimizer-config", - str(request.optimizer_config_path), + _display_path(request.optimizer_config_path), "--output-dir", - str(request.output_dir), + "$OUTPUT_DIR", "--run-id", request.run_id, ] target_paths = list(request.target_prompt_paths.items()) if set(request.target_prompt_paths) == {"system_prompt"}: - command.extend(["--prompt", str(request.target_prompt_paths["system_prompt"])]) + command.extend(["--prompt", _display_path(request.target_prompt_paths["system_prompt"])]) else: for name, path in target_paths: - command.extend(["--target-prompt", f"{name}={path}"]) + command.extend(["--target-prompt", f"{name}={_display_path(path)}"]) if request.trace: command.append("--trace") if request.mode == "sdk" and request.sdk_call_agent: command.extend(["--sdk-call-agent", request.sdk_call_agent]) if request.gate_config_path is not None: - command.extend(["--gate-config", str(request.gate_config_path)]) + command.extend(["--gate-config", _display_path(request.gate_config_path)]) if request.update_source: command.append("--update-source") - return shlex.join(command) + return " ".join(_powershell_arg(item) for item in command) + + +def _powershell_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _powershell_arg(value: str) -> str: + if re.fullmatch(r"[A-Za-z0-9_./:-]+", value): + return value + return _powershell_quote(value) def _initial_writeback_journal( *, + run_id: str, selected_candidate: str | None, update_source: bool, before_hashes: dict[str, str], + candidate_hashes: dict[str, str], + input_hashes: dict[str, str], ) -> dict[str, Any]: if selected_candidate is None: state = "rejected" @@ -613,10 +1352,14 @@ def _initial_writeback_journal( else: state = "pending" return { + "run_id": run_id, "state": state, + "report_phase": "pre_write", "requested": update_source, "selected_candidate": selected_candidate, "before_hashes": dict(before_hashes), + "candidate_hashes": dict(candidate_hashes), + "input_hashes": dict(input_hashes), "after_hashes": {}, "error": None, } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index cd554673..c44bc717 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -2,14 +2,18 @@ from __future__ import annotations +import ctypes import json -import re -import shutil +import hashlib +import os +import sys +import tempfile from dataclasses import dataclass from pathlib import Path from typing import Any from .attribution import summarize_failures +from .artifacts import validate_artifact_component from .schemas import CandidatePrompt from .schemas import CaseDelta from .schemas import CostSummary @@ -30,7 +34,6 @@ "--output-dir /tmp/eval-optimize-loop " "--fake-model --fake-judge --trace" ) -ARTIFACT_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") @dataclass(frozen=True) @@ -154,34 +157,16 @@ def prepare_run_artifacts( top_level_json=output_path / "optimization_report.json", top_level_markdown=output_path / "optimization_report.md", ) - paths.prewrite_json.write_text(report_to_json(report), encoding="utf-8") - paths.prewrite_markdown.write_text(render_markdown(report), encoding="utf-8") - paths.audit_json.write_text( - json.dumps( - to_jsonable(report.audit), - indent=2, - ensure_ascii=False, - sort_keys=True, - allow_nan=False, - ) - + "\n", - encoding="utf-8", - ) + _atomic_write_text(paths.prewrite_json, report_to_json(report)) + _atomic_write_text(paths.prewrite_markdown, render_markdown(report)) + _atomic_write_text(paths.audit_json, _json_text(report.audit)) + write_audit_artifacts(report, output_path) journal = dict(report.audit.get("writeback_journal", {})) if journal.get("state") == "pending": journal["state"] = "prepared" - paths.writeback_journal.write_text( - json.dumps( - to_jsonable(journal), - indent=2, - ensure_ascii=False, - sort_keys=True, - allow_nan=False, - ) - + "\n", - encoding="utf-8", - ) - write_audit_artifacts(report, output_path) + # This durable journal is the completion marker for the whole prepare phase, + # so it must be replaced only after every prerequisite artifact exists. + persist_writeback_journal(paths, journal) return paths @@ -193,49 +178,119 @@ def finalize_run_artifacts(report: OptimizationReport, paths: RunArtifactPaths) ) if paths.run_dir != expected_run_dir: raise ValueError("run artifact paths do not match report run_id") - paths.writeback_json.write_text( - json.dumps( - to_jsonable(report.writeback), - indent=2, - ensure_ascii=False, - sort_keys=True, - allow_nan=False, - ) - + "\n", - encoding="utf-8", - ) - paths.writeback_journal.write_text( - json.dumps( - to_jsonable(report.audit.get("writeback_journal", {})), - indent=2, - ensure_ascii=False, - sort_keys=True, - allow_nan=False, - ) - + "\n", - encoding="utf-8", - ) - paths.audit_json.write_text( + # The terminal outcome is authoritative and is persisted before convenience + # reports. A later rendering failure cannot erase knowledge of source state. + persist_writeback_outcome(report, paths) + _atomic_write_text(paths.audit_json, _json_text(report.audit)) + _atomic_write_text(paths.final_json, report_to_json(report)) + _atomic_write_text(paths.final_markdown, render_markdown(report)) + paths.output_dir.mkdir(parents=True, exist_ok=True) + _atomic_write_text(paths.top_level_json, report_to_json(report)) + _atomic_write_text(paths.top_level_markdown, render_markdown(report)) + write_audit_artifacts(report, paths.output_dir) + + +def persist_writeback_journal(paths: RunArtifactPaths, journal: dict[str, Any]) -> None: + """Atomically persist the authoritative writeback state machine record.""" + + _atomic_write_text(paths.writeback_journal, _json_text(journal)) + + +def persist_writeback_outcome(report: OptimizationReport, paths: RunArtifactPaths) -> None: + """Persist a terminal writeback outcome before any nonessential artifact.""" + + persist_writeback_journal(paths, dict(report.audit.get("writeback_journal", {}))) + _atomic_write_text(paths.writeback_json, _json_text(report.writeback)) + + +def report_to_json(report: OptimizationReport) -> str: + return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True, allow_nan=False) + "\n" + + +def _json_text(value: Any) -> str: + return ( json.dumps( - to_jsonable(report.audit), + to_jsonable(value), indent=2, ensure_ascii=False, sort_keys=True, allow_nan=False, ) - + "\n", - encoding="utf-8", + + "\n" ) - paths.final_json.write_text(report_to_json(report), encoding="utf-8") - paths.final_markdown.write_text(render_markdown(report), encoding="utf-8") - paths.output_dir.mkdir(parents=True, exist_ok=True) - paths.top_level_json.write_text(report_to_json(report), encoding="utf-8") - paths.top_level_markdown.write_text(render_markdown(report), encoding="utf-8") - write_audit_artifacts(report, paths.output_dir) -def report_to_json(report: OptimizationReport) -> str: - return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True, allow_nan=False) + "\n" +def _atomic_write_text(path: Path, content: str) -> None: + _atomic_write_bytes(path, content.encode("utf-8")) + + +def _atomic_write_bytes(path: Path, content: bytes) -> None: + """Durably replace a critical artifact without exposing a partial file.""" + + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + stream = os.fdopen(fd, "wb") + fd = -1 + with stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + _durable_replace(temp_path, path) + finally: + active_exception = sys.exc_info()[0] is not None + cleanup_error: OSError | None = None + if fd >= 0: + try: + os.close(fd) + except OSError as error: + cleanup_error = error + try: + temp_path.unlink() + except FileNotFoundError: + pass + except OSError as error: + if cleanup_error is None: + cleanup_error = error + if cleanup_error is not None and not active_exception: + raise cleanup_error + + +def _fsync_directory(directory: Path) -> None: + """Synchronize a POSIX directory entry after atomic replacement.""" + + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + fd = os.open(directory, flags) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _durable_replace(source: Path, target: Path) -> None: + """Atomically replace a file and wait for rename metadata to reach storage.""" + + if os.name == "nt": + move_file_ex = ctypes.WinDLL("kernel32", use_last_error=True).MoveFileExW + move_file_ex.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32] + move_file_ex.restype = ctypes.c_int + movefile_replace_existing = 0x00000001 + movefile_write_through = 0x00000008 + succeeded = move_file_ex( + str(source.resolve()), + str(target.resolve()), + movefile_replace_existing | movefile_write_through, + ) + if not succeeded: + raise ctypes.WinError(ctypes.get_last_error()) + return + os.replace(source, target) + _fsync_directory(target.parent) def render_markdown(report: OptimizationReport) -> str: @@ -320,7 +375,7 @@ def render_markdown(report: OptimizationReport) -> str: "", "## Failure Attribution Summary", "", - f"Total failed case evaluations: {summary['total_failed_cases']}", + f"Total failed case evaluations: {summary['total_failed_cases']}", "", "| category | count |", "| --- | ---: |", @@ -338,7 +393,7 @@ def render_markdown(report: OptimizationReport) -> str: lines.extend([ "", "## Cost And Audit", - "", + "", f"Total cost: {report.audit.get('cost', {}).get('total', 0):.3f}", f"Config hash: `{report.audit.get('config_hash', '')}`", f"Run id: `{report.run.get('run_id', '')}`", @@ -353,8 +408,8 @@ def render_markdown(report: OptimizationReport) -> str: candidate = record["candidate"] lines.extend([ f"### {candidate.candidate_id}", - "", - "```diff", + "", + "```diff", candidate.prompt_diff, "```", "", @@ -363,9 +418,9 @@ def render_markdown(report: OptimizationReport) -> str: lines.extend([ "## Reproducibility", "", - "```bash", - report.run.get("reproducibility_command") - or report.audit.get("reproducibility_command") + f"```{report.run.get('reproducibility_shell') or 'bash'}", + report.run.get("reproducibility_command") + or report.audit.get("reproducibility_command") or REPRODUCIBILITY_COMMAND, "```", "", @@ -378,16 +433,15 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None run_dir = output_path / "runs" / run_id run_dir.mkdir(parents=True, exist_ok=True) - input_paths = report.audit.get("input_paths", {}) - config_path = input_paths.get("optimizer") - if config_path and Path(config_path).is_file(): - shutil.copyfile(config_path, run_dir / "config.snapshot.json") - else: - (run_dir / "config.snapshot.json").write_text("{}", encoding="utf-8") - - (run_dir / "input_hashes.json").write_text( - json.dumps(report.audit.get("input_hashes", {}), indent=2, sort_keys=True) + "\n", - encoding="utf-8", + # Never copy the raw optimizer file: it may contain API keys. The original + # byte hash remains in input_hashes while this human-readable snapshot is redacted. + _atomic_write_text( + run_dir / "config.snapshot.json", + _json_text(report.audit.get("config_snapshot", {})), + ) + _atomic_write_text( + run_dir / "input_hashes.json", + _json_text(report.audit.get("input_hashes", {})), ) prompt_dir = run_dir / "candidate_prompts" @@ -397,26 +451,25 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None results_dir.mkdir(exist_ok=True) diffs_dir.mkdir(exist_ok=True) - for record in report.candidates: + for index, record in enumerate(report.candidates, start=1): candidate: CandidatePrompt = record["candidate"] - candidate_name = _safe_artifact_name(candidate.candidate_id) + candidate_name = _candidate_artifact_name(index, candidate.candidate_id) candidate_dir = prompt_dir / candidate_name candidate_dir.mkdir(exist_ok=True) for field_name, prompt_text in candidate.bundle().items(): field_artifact = _safe_artifact_name(str(field_name)) - (candidate_dir / f"{field_artifact}.txt").write_text( - prompt_text, - encoding="utf-8", - ) - (diffs_dir / f"{candidate_name}.diff").write_text(candidate.prompt_diff, encoding="utf-8") + _atomic_write_text(candidate_dir / f"{field_artifact}.txt", prompt_text) + _atomic_write_text(diffs_dir / f"{candidate_name}.diff", candidate.prompt_diff) for split_name in ("train_result", "validation_result"): split_result = record[split_name] split_artifact = _safe_artifact_name(str(split_result.split)) path = results_dir / f"{candidate_name}_{split_artifact}.json" - path.write_text( - json.dumps(to_jsonable(split_result), indent=2, sort_keys=True, allow_nan=False) + "\n", - encoding="utf-8", - ) + _atomic_write_text(path, _json_text(split_result)) + + +def _candidate_artifact_name(index: int, candidate_id: str) -> str: + digest = hashlib.sha256(candidate_id.encode("utf-8")).hexdigest()[:12] + return f"{index:03d}-{digest}" def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float) -> str: @@ -432,6 +485,4 @@ def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float) def _safe_artifact_name(name: str) -> str: - if name in {"", ".", ".."} or not ARTIFACT_NAME_RE.fullmatch(name): - raise ValueError(f"unsafe audit artifact name: {name!r}") - return name + return validate_artifact_component(name, context="audit artifact name") diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 2279501c..991d9aae 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -20,19 +20,23 @@ sys.path.insert(0, str(HERE)) try: # Package import during tests. + from .eval_loop.artifacts import validate_artifact_component from .eval_loop.backends import FakeBackend from .eval_loop.backends import SDKBackend from .eval_loop.config import _parse_gate_config from .eval_loop.config import parse_optimizer_config + from .eval_loop.config import resolve_effective_seed from .eval_loop.loader import read_json from .eval_loop.pipeline import PipelineRequest from .eval_loop.pipeline import execute_pipeline from .eval_loop.schemas import OptimizationReport except ImportError: # Direct script execution. + from eval_loop.artifacts import validate_artifact_component from eval_loop.backends import FakeBackend from eval_loop.backends import SDKBackend from eval_loop.config import _parse_gate_config from eval_loop.config import parse_optimizer_config + from eval_loop.config import resolve_effective_seed from eval_loop.loader import read_json from eval_loop.pipeline import PipelineRequest from eval_loop.pipeline import execute_pipeline @@ -166,14 +170,34 @@ def build_pipeline_request_and_backend( "--fake-judge or use --mode sdk with --sdk-call-agent module:function." ) + train_resolved = Path(train_path).resolve() + validation_resolved = Path(val_path).resolve() + if train_resolved == validation_resolved: + raise ValueError("train and validation evalset paths must be different") + target_prompt_paths = _parse_target_prompt_paths( target_prompts, default_prompt_path=prompt_path, ) effective_run_id = validate_run_id(run_id) if run_id is not None else _default_run_id(mode) + optimizer_payload = read_json(optimizer_config_path) + effective_seed = resolve_effective_seed( + optimizer_payload, + path=optimizer_config_path, + strict_legacy=mode == "fake", + ) + if mode == "fake" and backend is not None and hasattr(backend, "seed"): + backend_seed = getattr(backend, "seed") + if ( + isinstance(backend_seed, bool) + or not isinstance(backend_seed, int) + or backend_seed != effective_seed + ): + raise ValueError( + f"fake backend seed {backend_seed!r} does not match effective seed {effective_seed}" + ) if mode == "fake": - optimizer_payload = read_json(optimizer_config_path) optimizer_config = parse_optimizer_config( optimizer_payload, path=optimizer_config_path, @@ -183,12 +207,14 @@ def build_pipeline_request_and_backend( if gate_config_path is not None else optimizer_config.gate.to_dict() ) + gate_config_source = "file" if gate_config_path is not None else "optimizer" selected_backend = backend or FakeBackend( - seed=optimizer_config.seed, + seed=effective_seed, trace_enabled=trace, ) else: gate_config = _load_sdk_gate_config(gate_config_path) + gate_config_source = "file" if gate_config_path is not None else "request" if backend is None: if not sdk_call_agent: raise ValueError("sdk mode requires --sdk-call-agent module:function") @@ -213,6 +239,8 @@ def build_pipeline_request_and_backend( run_id=effective_run_id, sdk_call_agent=sdk_call_agent, gate_config_path=Path(gate_config_path) if gate_config_path is not None else None, + effective_seed=effective_seed, + gate_config_source=gate_config_source, ) return request, selected_backend @@ -225,11 +253,12 @@ def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, Any] path_text = "--gate-config" else: try: - payload = json.loads(Path(gate_config_path).read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - raise ValueError(f"--gate-config {gate_config_path}: invalid JSON: {exc}") from exc - if not isinstance(payload, dict): - raise ValueError(f"--gate-config {gate_config_path}: expected a JSON object") + payload = read_json(gate_config_path) + except ValueError as exc: + raise ValueError( + f"--gate-config {gate_config_path}: invalid JSON for gate numeric fields " + f"(including max_total_cost): {exc}" + ) from exc gate_payload = payload.get("gate", payload) path_text = str(gate_config_path) if gate_payload is None: @@ -252,6 +281,7 @@ def _parse_target_prompt_paths( if not target_prompts: return {"system_prompt": Path(default_prompt_path).resolve()} parsed: dict[str, str | Path] = {} + casefold_names: set[str] = set() resolved_paths: set[Path] = set() for item in target_prompts: if "=" not in item: @@ -263,14 +293,21 @@ def _parse_target_prompt_paths( f"--target-prompt field name {name!r} is invalid; " "use /^[A-Za-z_][A-Za-z0-9_]*$/" ) + try: + validate_artifact_component(name, context="target-prompt field") + except ValueError as error: + raise ValueError(f"--target-prompt field name {name!r} is invalid") from error if not path: raise ValueError("--target-prompt must use non-empty name=path values") if name in parsed: raise ValueError(f"--target-prompt duplicate field name {name!r}") + if name.casefold() in casefold_names: + raise ValueError("--target-prompt field names must be case-insensitively unique") resolved_path = Path(path).resolve() if resolved_path in resolved_paths: raise ValueError("--target-prompt fields must not reference the same resolved file") resolved_paths.add(resolved_path) + casefold_names.add(name.casefold()) parsed[name] = resolved_path return parsed @@ -280,9 +317,12 @@ def validate_run_id(run_id: str) -> str: if not isinstance(run_id, str): raise ValueError(f"--run-id value {run_id!r} must be a string") - if run_id in {"", ".", ".."} or not RUN_ID_RE.fullmatch(run_id): + if not RUN_ID_RE.fullmatch(run_id): raise ValueError(f"--run-id value {run_id!r} is invalid") - return run_id + try: + return validate_artifact_component(run_id, context="run_id") + except ValueError as error: + raise ValueError(f"--run-id value {run_id!r} is invalid") from error def _default_run_id(mode: str) -> str: diff --git a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py index 18c5cafe..e44c9633 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py +++ b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py @@ -6,12 +6,16 @@ import pytest from examples.optimization.eval_optimize_loop.eval_loop import schemas +from examples.optimization.eval_optimize_loop.eval_loop.backends import FakeBackend from examples.optimization.eval_optimize_loop.eval_loop.config import parse_optimizer_config from examples.optimization.eval_optimize_loop.eval_loop.config import validate_inputs from examples.optimization.eval_optimize_loop.eval_loop.loader import load_eval_cases from examples.optimization.eval_optimize_loop.eval_loop.loader import load_optimizer_config from examples.optimization.eval_optimize_loop.eval_loop.loader import read_json from examples.optimization.eval_optimize_loop.run_pipeline import _load_sdk_gate_config +from examples.optimization.eval_optimize_loop.run_pipeline import _parse_target_prompt_paths +from examples.optimization.eval_optimize_loop.run_pipeline import build_pipeline_request_and_backend +from examples.optimization.eval_optimize_loop.run_pipeline import validate_run_id def test_optimizer_config_defaults_metrics_and_gate(tmp_path: Path): @@ -82,6 +86,150 @@ def test_sdk_gate_config_allows_disabling_cost_gate(tmp_path: Path): assert gate["max_total_cost"] is None +@pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity"]) +def test_sdk_gate_config_rejects_non_standard_constants(tmp_path: Path, constant: str): + path = tmp_path / "gate.json" + path.write_text(f'{{"gate": {{"max_total_cost": {constant}}}}}', encoding="utf-8") + + with pytest.raises(ValueError, match="non-standard JSON constant"): + _load_sdk_gate_config(path) + + +def test_factory_rejects_same_resolved_train_and_validation_path(tmp_path: Path): + eval_path = tmp_path / "same.evalset.json" + eval_path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text('{"seed": 91}', encoding="utf-8") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="train and validation.*different"): + build_pipeline_request_and_backend( + train_path=eval_path, + val_path=eval_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + ) + + +@pytest.mark.parametrize("run_id", ["CON", "NUL.txt", "trailing.", "x" * 129]) +def test_run_id_rejects_windows_unsafe_or_oversized_components(run_id: str): + with pytest.raises(ValueError, match="run-id.*invalid"): + validate_run_id(run_id) + + +@pytest.mark.parametrize( + "targets", + [ + ["CON=one.txt"], + ["Foo=one.txt", "foo=two.txt"], + [f"{'x' * 129}=one.txt"], + ], +) +def test_target_prompt_fields_are_windows_safe_and_casefold_unique( + tmp_path: Path, + targets: list[str], +): + with pytest.raises(ValueError, match="target-prompt.*invalid|case-insensitively unique"): + _parse_target_prompt_paths(targets, default_prompt_path=tmp_path / "prompt.txt") + + +def test_nested_optimizer_seed_drives_backend_and_audit_request(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + for path in (train_path, val_path): + path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text('{"optimize": {"algorithm": {"seed": 7}}}', encoding="utf-8") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + request, backend = build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + ) + + assert backend.seed == 7 + assert request.effective_seed == 7 + + +def test_conflicting_top_level_and_nested_optimizer_seeds_are_rejected(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + for path in (train_path, val_path): + path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text( + '{"seed": 91, "optimize": {"algorithm": {"seed": 7}}}', + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="conflicting.*seed"): + build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + ) + + +def test_official_nested_seed_ignores_unrelated_string_top_level_seed(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + for path in (train_path, val_path): + path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text( + '{"seed": "sdk-owned-label", "optimize": {"algorithm": {"seed": 7}}}', + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + request, backend = build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + ) + + assert request.effective_seed == backend.seed == 7 + + +def test_factory_rejects_injected_fake_backend_seed_mismatch(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + for path in (train_path, val_path): + path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text('{"seed": 91}', encoding="utf-8") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="backend seed.*effective seed"): + build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + backend=FakeBackend(seed=7), + ) + + @pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity"]) def test_read_json_rejects_non_standard_constants(tmp_path: Path, constant: str): path = tmp_path / "invalid.json" diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py index f9434728..fe7013e5 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -103,12 +103,14 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert "Cost And Audit" in markdown run_dir = output_dir / "runs" / payload["run"]["run_id"] + overfit_artifact = payload["audit"]["candidate_artifacts"]["candidate_001_overfit"] + safe_artifact = payload["audit"]["candidate_artifacts"]["candidate_002_safe"] assert (run_dir / "config.snapshot.json").is_file() assert (run_dir / "input_hashes.json").is_file() - assert (run_dir / "candidate_prompts" / "candidate_001_overfit" / "system_prompt.txt").is_file() - assert (run_dir / "case_results" / "candidate_002_safe_validation.json").is_file() - assert (run_dir / "prompt_diffs" / "candidate_002_safe.diff").is_file() - assert (run_dir / "prompt_diffs" / "candidate_002_safe.diff").read_text(encoding="utf-8") == ( + assert (run_dir / "candidate_prompts" / overfit_artifact / "system_prompt.txt").is_file() + assert (run_dir / "case_results" / f"{safe_artifact}_validation.json").is_file() + assert (run_dir / "prompt_diffs" / f"{safe_artifact}.diff").is_file() + assert (run_dir / "prompt_diffs" / f"{safe_artifact}.diff").read_text(encoding="utf-8") == ( payload["candidates"][1]["candidate"]["prompt_diff"] ) @@ -211,4 +213,5 @@ def _normalized_payload(payload: dict) -> dict: normalized["run"].pop("reproducibility_command", None) normalized["audit"].pop("duration_seconds", None) normalized["audit"].pop("reproducibility_command", None) + normalized["audit"].get("writeback_journal", {}).pop("run_id", None) return normalized diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py index 69a84321..04b3fe3a 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py @@ -2,6 +2,8 @@ import hashlib import json +import math +import os from dataclasses import replace from pathlib import Path from typing import Any @@ -9,6 +11,8 @@ import pytest from examples.optimization.eval_optimize_loop.eval_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.eval_loop import report as report_module +from examples.optimization.eval_optimize_loop.eval_loop.backends import FakeBackend from examples.optimization.eval_optimize_loop.eval_loop.pipeline import PipelineRequest from examples.optimization.eval_optimize_loop.eval_loop.pipeline import execute_pipeline from examples.optimization.eval_optimize_loop.eval_loop.schemas import CandidatePrompt @@ -16,7 +20,10 @@ from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationRound from examples.optimization.eval_optimize_loop.eval_loop.schemas import WritebackResult +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ConcurrentPromptUpdateError +from examples.optimization.eval_optimize_loop.run_pipeline import build_pipeline_request_and_backend class RecordingBackend: @@ -26,10 +33,14 @@ def __init__( candidates: list[CandidatePrompt], results: dict[tuple[str, str], EvalResult], optimization_cost: CostSummary | None = None, + rounds: list[OptimizationRound] | None = None, + raw_summary: dict[str, object] | None = None, ) -> None: self.candidates = candidates self.results = results self.optimization_cost = optimization_cost or CostSummary() + self.rounds = list(rounds or []) + self.raw_summary = raw_summary or {"status": "SUCCEEDED"} self.calls: list[tuple[Any, ...]] = [] self.failure_summary: dict[str, object] | None = None @@ -61,9 +72,9 @@ async def optimize_candidates( self.failure_summary = failure_summary return OptimizationResult( candidates=self.candidates, - rounds=[], + rounds=self.rounds, cost=self.optimization_cost, - raw_summary={"status": "SUCCEEDED"}, + raw_summary=self.raw_summary, ) @@ -90,9 +101,7 @@ async def test_execute_pipeline_uses_train_only_evidence_and_full_candidate_reev assert report.selected_candidate == "candidate_a" assert report.audit["duration_seconds"] > 0 assert report.audit["config_snapshot"] == {"seed": 91} - assert report.audit["input_hashes"]["target_prompts"]["system_prompt"] == hashlib.sha256( - b"baseline\n" - ).hexdigest() + assert report.audit["input_hashes"]["target_prompts"]["system_prompt"] == hashlib.sha256(b"baseline\n").hexdigest() artifact_dirs = [call[4] for call in backend.calls if call[0] == "evaluate"] assert len(artifact_dirs) == len(set(artifact_dirs)) == 6 assert report.audit["sdk_result_availability"]["full_train_eval_result"] is True @@ -116,6 +125,7 @@ async def test_rejected_candidate_never_changes_source_when_writeback_requested( assert report.writeback.before_hashes == { "system_prompt": hashlib.sha256(before).hexdigest(), } + assert report.audit["writeback_journal"]["report_phase"] == "final" assert (tmp_path / "prompt.txt").read_bytes() == before @@ -165,13 +175,30 @@ def finalize(report, paths): assert paths is artifact_paths assert report.writeback.status == "applied" + def persist_journal(paths, journal): + events.append(f"journal:{journal['state']}") + assert paths is artifact_paths + + def persist_outcome(report, paths): + events.append(f"outcome:{report.audit['writeback_journal']['state']}") + assert paths is artifact_paths + monkeypatch.setattr(pipeline_module, "prepare_run_artifacts", prepare) monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", commit) monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", finalize) + monkeypatch.setattr(pipeline_module, "persist_writeback_journal", persist_journal) + monkeypatch.setattr(pipeline_module, "persist_writeback_outcome", persist_outcome) report = await execute_pipeline(request, evaluator=backend, optimizer=backend) - assert events == ["prepare", "commit", "finalize"] + assert events == [ + "prepare", + "journal:committing", + "commit", + "journal:applied", + "outcome:applied", + "finalize", + ] assert report.writeback.status == "applied" @@ -266,11 +293,9 @@ async def test_optimizer_cost_is_counted_once_across_candidates(tmp_path: Path): @pytest.mark.parametrize( ("baseline_case_ids", "candidate_case_ids", "reason_fragment"), [ - (["a", "a"], ["a"], "duplicate case IDs"), (["a"], ["a", "a"], "duplicate case IDs"), (["a", "b"], ["a"], "case ID set mismatch"), (["a"], ["a", "extra"], "case ID set mismatch"), - ([], [], "empty case results"), ], ) async def test_incomparable_case_results_are_rejected_without_delta_error( @@ -280,6 +305,18 @@ async def test_incomparable_case_results_are_rejected_without_delta_error( reason_fragment: str, ): request = _request(tmp_path) + for dataset_path in (Path(request.train_path), Path(request.validation_path)): + dataset_path.write_text( + json.dumps( + { + "eval_cases": [ + {"eval_id": case_id, "session_input": {"state": {}}} + for case_id in dict.fromkeys(baseline_case_ids) + ] + } + ), + encoding="utf-8", + ) candidate = CandidatePrompt("candidate_a", "safe prompt", "safe", "diff") results = { ("baseline", "train"): _eval("baseline", "train", baseline_case_ids, 0.0), @@ -297,6 +334,18 @@ async def test_incomparable_case_results_are_rejected_without_delta_error( assert any(reason_fragment in reason for reason in report.gate_decisions[0].reasons) +@pytest.mark.asyncio +async def test_empty_evalsets_are_rejected_before_backend_calls(tmp_path: Path): + request = _request(tmp_path) + Path(request.train_path).write_text('{"eval_cases": []}', encoding="utf-8") + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="train evalset.*must not be empty"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + + @pytest.mark.asyncio async def test_duplicate_candidate_ids_are_rejected(tmp_path: Path): request = _request(tmp_path) @@ -368,16 +417,6 @@ async def test_finalize_failure_leaves_explicit_prepared_writeback_journal( request = _request(tmp_path, update_source=True) backend = _safe_backend(candidate_count=1) - monkeypatch.setattr( - pipeline_module, - "commit_prompt_bundle", - lambda snapshot, prompts: WritebackResult( - status="applied", - before_hashes=snapshot.hashes(), - after_hashes=snapshot.hashes(), - ), - ) - def fail_finalize(report, paths): raise OSError("final report unavailable") @@ -391,7 +430,113 @@ def fail_finalize(report, paths): journal = json.loads((run_dir / "writeback_journal.json").read_text(encoding="utf-8")) assert prewrite["audit"]["writeback_journal"]["state"] == "pending" assert "pending" in prewrite["writeback"]["error"] - assert journal["state"] == "prepared" + assert journal["state"] == "applied" + assert (tmp_path / "prompt.txt").read_bytes() == b"safe prompt" + assert journal["after_hashes"] == { + "system_prompt": hashlib.sha256(b"safe prompt").hexdigest(), + } + + +@pytest.mark.asyncio +async def test_writeback_conflict_is_caught_and_durably_reported( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + + def conflict(snapshot, prompts): + raise ConcurrentPromptUpdateError("source changed") + + monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", conflict) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert report.writeback.status == "rejected" + assert "source changed" in (report.writeback.error or "") + assert journal["state"] == "conflict" + assert journal["error"] == report.writeback.error + + +@pytest.mark.asyncio +async def test_real_source_drift_between_prepare_and_commit_is_terminal_conflict( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + original_prepare = pipeline_module.prepare_run_artifacts + + def prepare_then_external_change(report, output_dir): + paths = original_prepare(report, output_dir) + (tmp_path / "prompt.txt").write_bytes(b"external writer\n") + return paths + + monkeypatch.setattr(pipeline_module, "prepare_run_artifacts", prepare_then_external_change) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert report.writeback.status == "rejected" + assert report.writeback.after_hashes == { + "system_prompt": hashlib.sha256(b"external writer\n").hexdigest(), + } + assert journal["state"] == "conflict" + assert journal["after_hashes"] == report.writeback.after_hashes + assert (tmp_path / "prompt.txt").read_bytes() == b"external writer\n" + + +@pytest.mark.asyncio +async def test_terminal_journal_survives_nonessential_writeback_artifact_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + + def fail_outcome(report, paths): + raise OSError("writeback convenience artifact unavailable") + + monkeypatch.setattr(pipeline_module, "persist_writeback_outcome", fail_outcome) + + with pytest.raises(OSError, match="convenience artifact"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert (tmp_path / "prompt.txt").read_bytes() == b"safe prompt" + assert journal["state"] == "applied" + assert journal["after_hashes"] == { + "system_prompt": hashlib.sha256(b"safe prompt").hexdigest(), + } + + +@pytest.mark.asyncio +async def test_writeback_journal_is_committing_before_source_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + + def commit(snapshot, prompts): + journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert journal["state"] == "committing" + return WritebackResult( + status="applied", + before_hashes=snapshot.hashes(), + after_hashes={name: hashlib.sha256(prompts[name].encode("utf-8")).hexdigest() for name in prompts}, + ) + + monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", commit) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.writeback.status == "applied" @pytest.mark.asyncio @@ -429,6 +574,10 @@ async def test_failed_writeback_is_persisted_in_final_report_and_journal( @pytest.mark.asyncio async def test_standard_evalset_protected_metadata_is_applied_to_gate(tmp_path: Path): request = _request(tmp_path) + Path(request.train_path).write_text( + json.dumps({"eval_cases": [{"eval_id": "train", "session_input": {"state": {}}}]}), + encoding="utf-8", + ) Path(request.validation_path).write_text( """{ "eval_set_id": "validation", @@ -470,6 +619,514 @@ async def test_standard_evalset_protected_metadata_is_applied_to_gate(tmp_path: assert report.audit["gate_config_snapshot"]["protected_case_ids"] == ["protected"] +@pytest.mark.asyncio +async def test_baseline_results_must_cover_every_snapshotted_dataset_case(tmp_path: Path): + request = _request(tmp_path, update_source=True) + Path(request.validation_path).write_text( + json.dumps( + { + "eval_cases": [ + {"eval_id": "validation_case", "session_input": {"state": {}}}, + { + "eval_id": "protected_missing", + "session_input": { + "state": {"eval_optimize_protected": True} + }, + }, + ] + } + ), + encoding="utf-8", + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="baseline.*validation.*dataset case IDs"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert (tmp_path / "prompt.txt").read_bytes() == b"baseline\n" + + +@pytest.mark.asyncio +async def test_backend_neutral_core_rejects_same_resolved_train_and_validation_path(tmp_path: Path): + request = _request(tmp_path) + request = replace(request, validation_path=request.train_path) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="train and validation.*different"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field_names", [["Foo", "foo"], ["CON"]]) +async def test_backend_neutral_core_rejects_unsafe_or_colliding_target_fields( + tmp_path: Path, + field_names: list[str], +): + request = _request(tmp_path) + target_paths: dict[str, Path] = {} + for index, field_name in enumerate(field_names): + prompt_path = tmp_path / f"prompt-{index}.txt" + prompt_path.write_text(f"prompt {index}", encoding="utf-8") + target_paths[field_name] = prompt_path + request = replace(request, target_prompt_paths=target_paths) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="target prompt field.*unsafe|case-insensitively unique"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + + +@pytest.mark.asyncio +async def test_backend_neutral_core_rejects_target_fields_for_same_resolved_file( + tmp_path: Path, +): + request = _request(tmp_path) + prompt_path = request.target_prompt_paths["system_prompt"] + request = replace( + request, + target_prompt_paths={"system_prompt": prompt_path, "router_prompt": prompt_path}, + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="target prompt fields.*same resolved file"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + + +@pytest.mark.asyncio +async def test_backend_neutral_core_rejects_fake_backend_seed_mismatch(tmp_path: Path): + request = _request(tmp_path) + backend = FakeBackend(seed=7) + + with pytest.raises(ValueError, match="backend seed.*effective seed"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_fake_gate_is_derived_from_immutable_optimizer_snapshot(tmp_path: Path): + base_request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + Path(base_request.optimizer_config_path).write_text( + json.dumps({"seed": 91, "gate": {"max_total_cost": None}}), + encoding="utf-8", + ) + request, selected_backend = build_pipeline_request_and_backend( + train_path=base_request.train_path, + val_path=base_request.validation_path, + optimizer_config_path=base_request.optimizer_config_path, + prompt_path=base_request.target_prompt_paths["system_prompt"], + output_dir=base_request.output_dir, + mode="fake", + run_id="gate_snapshot_run", + backend=backend, + ) + Path(base_request.optimizer_config_path).write_text( + json.dumps({"seed": 91, "gate": {"max_total_cost": 0.0}}), + encoding="utf-8", + ) + + report = await execute_pipeline( + request, + evaluator=selected_backend, + optimizer=selected_backend, + ) + + assert report.selected_candidate is None + assert report.audit["gate_config_snapshot"]["max_total_cost"] == 0.0 + + +@pytest.mark.asyncio +async def test_configured_protected_case_ids_must_exist_in_validation_metadata(tmp_path: Path): + request = _request( + tmp_path, + gate_config={ + "min_val_score_improvement": 0.0, + "protected_case_ids": ["missing"], + "max_score_drop_per_case": 1.0, + "max_total_cost": None, + }, + ) + Path(request.validation_path).write_text( + json.dumps({"eval_cases": [{"eval_id": "present", "session_input": {"state": {}}}]}), + encoding="utf-8", + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="missing validation cases.*missing"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload", + [ + {"eval_cases": [{"eval_id": 7, "session_input": {"state": {}}}]}, + {"eval_cases": [{"eval_id": "", "session_input": {"state": {}}}]}, + {"eval_cases": [{"eval_id": "case", "session_input": {"state": {"eval_optimize_protected": "yes"}}}]}, + {"eval_cases": [{"eval_id": "case", "session_input": {"state": {"eval_optimize_protected": None}}}]}, + {"cases": [{"case_id": 7, "protected": False}]}, + {"cases": [{"case_id": "case", "protected": "yes"}]}, + {"cases": [{"case_id": "case", "protected": 1}]}, + ], +) +async def test_validation_metadata_rejects_coerced_ids_and_protected_flags( + tmp_path: Path, + payload: dict[str, object], +): + request = _request(tmp_path) + Path(request.validation_path).write_text(json.dumps(payload), encoding="utf-8") + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="case ID|protected.*boolean"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_validation_metadata_rejects_ambiguous_dual_schema(tmp_path: Path): + request = _request(tmp_path) + Path(request.validation_path).write_text( + json.dumps({"eval_cases": [], "cases": []}), + encoding="utf-8", + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="exactly one.*eval_cases.*cases"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("candidate_ids", [["CON"], ["candidate."], ["Candidate", "candidate"], ["bad/name"]]) +async def test_candidate_ids_are_artifact_safe_and_casefold_unique( + tmp_path: Path, + candidate_ids: list[str], +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=0) + backend.candidates = [ + CandidatePrompt(candidate_id, "safe prompt", "safe", "diff") for candidate_id in candidate_ids + ] + + with pytest.raises(ValueError, match="candidate_id"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_candidate_prompts_must_be_valid_utf8_before_evaluation(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=0) + backend.candidates = [ + CandidatePrompt("candidate", "bad\ud800prompt", "safe", "diff") + ] + + with pytest.raises(ValueError, match="valid UTF-8"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert [call[:3] for call in backend.calls] == [ + ("evaluate", "baseline", "train"), + ("evaluate", "baseline", "validation"), + ("optimize", {"system_prompt": "baseline\n"}, Path(request.output_dir) / "runs" / request.run_id / "optimizer"), + ] + + +@pytest.mark.asyncio +async def test_eval_results_reject_non_finite_scores_and_negative_costs(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + baseline = backend.results[("baseline", "train")] + backend.results[("baseline", "train")] = replace( + baseline, + cost=-0.01, + cases=[replace(baseline.cases[0], score=math.nan)], + ) + + with pytest.raises(ValueError, match="finite|non-negative"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["result_score", "result_cost", "case_metric"]) +async def test_eval_results_reject_boolean_numeric_values(tmp_path: Path, field: str): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + baseline = backend.results[("baseline", "train")] + if field == "result_score": + malformed = replace(baseline, score=True) + elif field == "result_cost": + malformed = replace(baseline, cost=True) + else: + malformed = replace( + baseline, + cases=[replace(baseline.cases[0], metrics={"quality": True})], + ) + backend.results[("baseline", "train")] = malformed + + with pytest.raises(ValueError, match="finite number"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_complete_cost_summary_must_equal_components(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend( + candidate_count=1, + optimization_cost=CostSummary(optimizer=0.1, evaluator=0.2, agent=0.3, total=9.0, complete=True), + ) + + with pytest.raises(ValueError, match="total.*components"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_optimizer_rounds_reject_invalid_numeric_values(tmp_path: Path): + request = _request(tmp_path) + invalid_round = OptimizationRound( + round_id=1, + candidate_id="candidate_a", + prompts={"system_prompt": "safe prompt"}, + rationale="bad duration", + metrics={"quality": math.inf}, + cost=CostSummary(), + duration_seconds=-1.0, + ) + backend = _safe_backend(candidate_count=1) + backend.rounds = [invalid_round] + + with pytest.raises(ValueError, match="round.*finite|duration.*non-negative"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bad_value", [Path("private.txt"), {"not-hashable"}, object()]) +async def test_optimizer_raw_summary_must_be_json_compatible( + tmp_path: Path, + bad_value: object, +): + request = _request(tmp_path, update_source=True) + delegate = _safe_backend(candidate_count=1) + backend = RecordingBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + raw_summary={"bad": bad_value}, + ) + + with pytest.raises(ValueError, match="JSON-compatible"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert (tmp_path / "prompt.txt").read_bytes() == b"baseline\n" + + +class SnapshotMutationBackend(RecordingBackend): + def __init__(self, *, request: PipelineRequest, delegate: RecordingBackend) -> None: + super().__init__( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + self.request = request + self.dataset_paths: list[Path] = [] + self.optimizer_paths: list[tuple[Path, Path, Path]] = [] + self._mutated = False + + async def evaluate(self, **kwargs: Any) -> EvalResult: + dataset_path = Path(kwargs["dataset_path"]) + self.dataset_paths.append(dataset_path) + expected_id = "train_case" if kwargs["split"] == "train" else "validation_case" + payload = json.loads(dataset_path.read_text(encoding="utf-8")) + assert [case["eval_id"] for case in payload["eval_cases"]] == [expected_id] + result = await super().evaluate(**kwargs) + if not self._mutated: + Path(self.request.train_path).write_text('{"eval_cases": [{"eval_id": "changed"}]}', encoding="utf-8") + Path(self.request.optimizer_config_path).write_text('{"seed": 999}', encoding="utf-8") + if self.request.gate_config_path is not None: + Path(self.request.gate_config_path).write_text('{"gate": {"max_total_cost": 0}}', encoding="utf-8") + self._mutated = True + return result + + async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: + paths = ( + Path(kwargs["train_path"]), + Path(kwargs["validation_path"]), + Path(kwargs["config_path"]), + ) + self.optimizer_paths.append(paths) + assert json.loads(paths[0].read_text(encoding="utf-8"))["eval_cases"][0]["eval_id"] == "train_case" + assert json.loads(paths[1].read_text(encoding="utf-8"))["eval_cases"][0]["eval_id"] == "validation_case" + assert json.loads(paths[2].read_text(encoding="utf-8"))["seed"] == 91 + return await super().optimize_candidates(**kwargs) + + +class SnapshotTamperingBackend(RecordingBackend): + async def evaluate(self, **kwargs: Any) -> EvalResult: + result = await super().evaluate(**kwargs) + if kwargs["prompt_id"] == "baseline" and kwargs["split"] == "train": + Path(kwargs["dataset_path"]).write_text( + '{"eval_cases": [{"eval_id": "tampered"}]}', + encoding="utf-8", + ) + return result + + +@pytest.mark.asyncio +async def test_backend_cannot_mutate_immutable_input_snapshot(tmp_path: Path): + request = _request(tmp_path, update_source=True) + delegate = _safe_backend(candidate_count=1) + backend = SnapshotTamperingBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + + with pytest.raises(ValueError, match="immutable input snapshot.*train"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert (tmp_path / "prompt.txt").read_bytes() == b"baseline\n" + assert list(tmp_path.glob(".*.snapshot-*")) == [] + + +@pytest.mark.asyncio +async def test_pipeline_uses_immutable_entry_snapshots_and_rejects_original_input_drift(tmp_path: Path): + request = _request(tmp_path, update_source=True) + gate_path = tmp_path / "gate.json" + gate_path.write_text('{"gate": {"max_total_cost": null}}', encoding="utf-8") + request = replace(request, gate_config_path=gate_path) + backend = SnapshotMutationBackend(request=request, delegate=_safe_backend(candidate_count=1)) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + original_paths = { + Path(request.train_path).resolve(), + Path(request.validation_path).resolve(), + Path(request.optimizer_config_path).resolve(), + } + used_paths = {path.resolve() for path in backend.dataset_paths} + used_paths.update(path.resolve() for triple in backend.optimizer_paths for path in triple) + assert used_paths.isdisjoint(original_paths) + assert all(".snapshot-" in path.name for path in used_paths) + assert all( + path.parent + in { + Path(request.train_path).resolve().parent, + Path(request.validation_path).resolve().parent, + Path(request.optimizer_config_path).resolve().parent, + } + for path in used_paths + ) + assert report.writeback.status == "rejected" + assert report.audit["writeback_journal"]["state"] == "conflict" + assert "input changed" in (report.writeback.error or "") + assert (tmp_path / "prompt.txt").read_bytes() == b"baseline\n" + assert list(tmp_path.glob(".*.snapshot-*")) == [] + + +@pytest.mark.asyncio +async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_path: Path): + request = _request(tmp_path) + config_payload = { + "seed": 91, + "api_key": "plain-api-key", + "nested": { + "access_token": "plain-token", + "credentials": {"password": "plain-password"}, + "github_token": "plain-github-token", + "private_key": "plain-private-key", + "cache_path": str(tmp_path / "private-cache"), + }, + } + Path(request.optimizer_config_path).write_text(json.dumps(config_payload), encoding="utf-8") + backend = _safe_backend(candidate_count=1) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + payload = json.dumps(report, default=lambda value: value.__dict__, sort_keys=True) + + assert "plain-api-key" not in payload + assert "plain-token" not in payload + assert "plain-password" not in payload + assert "plain-github-token" not in payload + assert "plain-private-key" not in payload + assert str(tmp_path.resolve()) not in payload + assert report.run["reproducibility_shell"] == "powershell" + assert "$EXTERNAL" in report.run["reproducibility_command"] + assert ( + report.audit["config_hash"] + == hashlib.sha256( + json.dumps(config_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + ) + persisted = (Path(request.output_dir) / "optimization_report.json").read_text(encoding="utf-8") + persisted_markdown = (Path(request.output_dir) / "optimization_report.md").read_text(encoding="utf-8") + config_snapshot = (Path(request.output_dir) / "runs" / request.run_id / "config.snapshot.json").read_text( + encoding="utf-8" + ) + assert "plain-api-key" not in persisted + config_snapshot + assert "plain-token" not in persisted + config_snapshot + assert "plain-password" not in persisted + config_snapshot + assert "plain-github-token" not in persisted + config_snapshot + assert "plain-private-key" not in persisted + config_snapshot + assert "```powershell" in persisted_markdown + assert str(tmp_path.resolve()) not in persisted_markdown + assert hashlib.sha256(config_snapshot.encode("utf-8")).hexdigest() == report.audit[ + "redacted_config_snapshot_sha256" + ] + assert report.audit["config_file_sha256"] == hashlib.sha256( + Path(request.optimizer_config_path).read_bytes() + ).hexdigest() + + +def test_atomic_artifact_write_fsyncs_before_durable_replace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + path = tmp_path / "artifact.json" + events: list[tuple[str, object]] = [] + real_fsync = os.fsync + real_durable_replace = report_module._durable_replace + + def tracked_fsync(fd: int) -> None: + events.append(("fsync", fd)) + real_fsync(fd) + + def tracked_durable_replace(source: Path, target: Path) -> None: + events.append(("durable_replace", Path(target))) + real_durable_replace(source, target) + + monkeypatch.setattr(report_module.os, "fsync", tracked_fsync) + monkeypatch.setattr(report_module, "_durable_replace", tracked_durable_replace) + + report_module._atomic_write_text(path, "durable\n") + + replace_index = next(index for index, event in enumerate(events) if event[0] == "durable_replace") + assert any(event[0] == "fsync" for event in events[:replace_index]) + assert path.read_text(encoding="utf-8") == "durable\n" + assert not list(tmp_path.glob("*.tmp")) + + +@pytest.mark.asyncio +async def test_prepare_writes_completion_journal_after_all_other_artifacts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + writes: list[str] = [] + original = report_module._atomic_write_text + + def tracked(path: Path, content: str) -> None: + writes.append(Path(path).name) + original(Path(path), content) + + monkeypatch.setattr(report_module, "_atomic_write_text", tracked) + + report_module.prepare_run_artifacts(report, tmp_path / "prepare-order") + + assert writes[-1] == "writeback_journal.json" + + @pytest.mark.asyncio @pytest.mark.parametrize( ("identity_error", "message"), @@ -562,8 +1219,14 @@ def _request( validation_path = tmp_path / "validation.evalset.json" config_path = tmp_path / "optimizer.json" prompt_path = tmp_path / "prompt.txt" - train_path.write_text('{"eval_cases": []}', encoding="utf-8") - validation_path.write_text('{"eval_cases": []}', encoding="utf-8") + train_path.write_text( + '{"eval_cases": [{"eval_id": "train_case", "session_input": {"state": {}}}]}', + encoding="utf-8", + ) + validation_path.write_text( + '{"eval_cases": [{"eval_id": "validation_case", "session_input": {"state": {}}}]}', + encoding="utf-8", + ) config_path.write_text('{"seed": 91}', encoding="utf-8") prompt_path.write_bytes(b"baseline\n") return PipelineRequest( @@ -572,7 +1235,8 @@ def _request( optimizer_config_path=config_path, output_dir=tmp_path / "out", target_prompt_paths={"system_prompt": prompt_path}, - gate_config=gate_config or { + gate_config=gate_config + or { "min_val_score_improvement": 0.0, "allow_new_hard_fail": False, "protected_case_ids": [], diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 4c8893d7..01c402df 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -1240,7 +1240,8 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert "sdk_best (accepted)" in markdown assert "complete AgentEvaluator-compatible reevaluation" in markdown assert (output_dir / "runs" / "sdk_test_run" / "input_hashes.json").is_file() - assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / "sdk_best.diff").is_file() + candidate_artifact = report.audit["candidate_artifacts"]["sdk_best"] + assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / f"{candidate_artifact}.diff").is_file() assert calls["update_source"] is False assert calls["output_dir"].endswith("runs\\sdk_test_run\\optimizer") or calls[ "output_dir" @@ -1250,7 +1251,8 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert "--sdk-call-agent fake_call_agent_module:call_agent" in command command_args = shlex.split(command) gate_index = command_args.index("--gate-config") - assert command_args[gate_index + 1] == str(gate_path) + assert command_args[gate_index + 1] == "$EXTERNAL/wrapper_gate.json" + assert str(gate_path.parent.resolve()) not in command def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp_path: Path, monkeypatch): @@ -1352,7 +1354,7 @@ def test_run_pipeline_mode_sdk_default_run_ids_are_unique(tmp_path: Path, monkey assert (tmp_path / "sdk_run" / "runs" / second.run["run_id"]).is_dir() -def test_run_pipeline_mode_sdk_explicit_run_id_stays_stable(tmp_path: Path, monkeypatch): +def test_run_pipeline_mode_sdk_explicit_run_id_is_stable_and_exclusive(tmp_path: Path, monkeypatch): _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") first = run_pipeline( @@ -1365,19 +1367,19 @@ def test_run_pipeline_mode_sdk_explicit_run_id_stays_stable(tmp_path: Path, monk sdk_call_agent="fake_call_agent_module:call_agent", run_id="valid_20260704-1.ok", ) - second = run_pipeline( - mode="sdk", - train_path=DEFAULT_TRAIN, - val_path=DEFAULT_VAL, - optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, - prompt_path=DEFAULT_PROMPT, - output_dir=tmp_path / "sdk_run", - sdk_call_agent="fake_call_agent_module:call_agent", - run_id="valid_20260704-1.ok", - ) + with pytest.raises(ValueError, match="run_id.*already exists"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id="valid_20260704-1.ok", + ) assert first.run["run_id"] == "valid_20260704-1.ok" - assert second.run["run_id"] == "valid_20260704-1.ok" def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_gate( @@ -1662,8 +1664,11 @@ def test_run_pipeline_mode_sdk_does_not_pass_wrapper_gate_config_to_agent_optimi gate_config_path=gate_path, ) - assert Path(calls["config_path"]).resolve() == optimizer_path.resolve() - assert "gate" not in json.loads(Path(calls["config_path"]).read_text(encoding="utf-8")) + optimizer_snapshot = Path(calls["config_path"]) + assert optimizer_snapshot.resolve() != optimizer_path.resolve() + assert optimizer_snapshot.parent.resolve() == optimizer_path.parent.resolve() + assert optimizer_snapshot.name.endswith(f".snapshot-{optimizer_path.name}") + assert "gate" not in calls["config_payload"] assert json.loads(gate_path.read_text(encoding="utf-8"))["gate"]["max_total_cost"] == 0.05 @@ -1717,13 +1722,14 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: "skill_prompt", } run_dir = tmp_path / "sdk_run" / "runs" / "sdk_multi_target" - assert (run_dir / "candidate_prompts" / "sdk_best" / "system_prompt.txt").read_text( + candidate_artifact = report.audit["candidate_artifacts"]["sdk_best"] + assert (run_dir / "candidate_prompts" / candidate_artifact / "system_prompt.txt").read_text( encoding="utf-8" ) == "optimized system" - assert (run_dir / "candidate_prompts" / "sdk_best" / "router_prompt.txt").read_text( + assert (run_dir / "candidate_prompts" / candidate_artifact / "router_prompt.txt").read_text( encoding="utf-8" ) == "optimized router" - assert (run_dir / "candidate_prompts" / "sdk_best" / "skill_prompt.txt").read_text( + assert (run_dir / "candidate_prompts" / candidate_artifact / "skill_prompt.txt").read_text( encoding="utf-8" ) == "optimized skill" input_hashes = json.loads((run_dir / "input_hashes.json").read_text(encoding="utf-8")) @@ -1738,7 +1744,8 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: command = report.run["reproducibility_command"] assert "--sdk-call-agent fake_call_agent_module:call_agent" in command assert "--target-prompt" in command - assert f"router_prompt={router_path}" in command + assert "router_prompt=$EXTERNAL/router.txt" in command + assert str(tmp_path.resolve()) not in command assert "--gate-config" in command @@ -1812,6 +1819,9 @@ class FakeAgentOptimizer: @staticmethod async def optimize(**kwargs): calls.update(kwargs) + config_path = Path(kwargs["config_path"]) + if config_path.is_file(): + calls["config_payload"] = json.loads(config_path.read_text(encoding="utf-8")) if result_override is not None: return result_override if write_source_when_requested and kwargs.get("update_source"): From a5d02803cb31cd21aecc0baea0f2916ad18898f1 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 02:45:15 +0800 Subject: [PATCH 26/34] fix(examples): satisfy eval optimize review contracts --- .../eval_optimize_loop/eval_loop/backends.py | 1 + .../eval_optimize_loop/eval_loop/pipeline.py | 318 +++++++++++++--- .../eval_optimize_loop/eval_loop/report.py | 68 +++- .../eval_optimize_loop/eval_loop/schemas.py | 7 +- .../eval_optimize_loop/eval_loop/writeback.py | 12 +- .../tests/test_pipeline_orchestration.py | 338 ++++++++++++++++++ .../tests/test_sdk_backend.py | 15 +- .../tests/test_writeback.py | 4 +- 8 files changed, 686 insertions(+), 77 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 5b3263a2..414b9606 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -442,6 +442,7 @@ async def optimize_candidates( optimizer=total_llm_cost, total=total_llm_cost, complete=False, + reported_optimizer_cost=total_llm_cost, ) result = OptimizationResult( candidates=candidates, diff --git a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py index 689a3d3e..114979af 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py @@ -9,9 +9,11 @@ import re import tempfile import time +from collections.abc import Mapping from dataclasses import dataclass from dataclasses import replace from pathlib import Path +from types import MappingProxyType from typing import Any from .attribution import summarize_failures @@ -33,12 +35,14 @@ from .schemas import CandidatePrompt from .schemas import CostSummary from .schemas import EvalResult +from .schemas import GateDecision from .schemas import OptimizationReport from .schemas import OptimizationResult from .schemas import WritebackResult from .schemas import to_jsonable from .writeback import commit_prompt_bundle from .writeback import ConcurrentPromptUpdateError +from .writeback import PromptRestorationError from .writeback import snapshot_prompt_files _REPOSITORY_ROOT = Path(__file__).resolve().parents[4] @@ -167,7 +171,7 @@ async def _execute_with_snapshots( input_snapshots: _InputSnapshots, ) -> OptimizationReport: prompt_snapshot = snapshot_prompt_files(request.target_prompt_paths) - baseline_prompts = _decode_snapshot(prompt_snapshot) + baseline_prompts = MappingProxyType(_decode_snapshot(prompt_snapshot)) raw_config = read_json(input_snapshots.path("optimizer")) effective_seed = resolve_effective_seed( raw_config, @@ -198,7 +202,7 @@ async def _execute_with_snapshots( _verify_snapshot_integrity(input_snapshots, "train") baseline_train = await evaluator.evaluate( prompt_id="baseline", - prompts=baseline_prompts, + prompts=dict(baseline_prompts), dataset_path=train_path, split="train", trace=request.trace, @@ -214,7 +218,7 @@ async def _execute_with_snapshots( _verify_snapshot_integrity(input_snapshots, "validation") baseline_validation = await evaluator.evaluate( prompt_id="baseline", - prompts=baseline_prompts, + prompts=dict(baseline_prompts), dataset_path=validation_path, split="validation", trace=request.trace, @@ -231,7 +235,7 @@ async def _execute_with_snapshots( failure_summary = summarize_failures([baseline_train]) _verify_snapshot_integrity(input_snapshots, "train", "validation", "optimizer") optimization = await optimizer.optimize_candidates( - baseline_prompts=baseline_prompts, + baseline_prompts=dict(baseline_prompts), baseline_train=baseline_train, failure_summary=failure_summary, train_path=train_path, @@ -249,45 +253,117 @@ async def _execute_with_snapshots( gate = AcceptanceGate(effective_gate_config) baseline_evaluator_cost = round(baseline_train.cost + baseline_validation.cost, 6) explicit_evaluator_cost = baseline_evaluator_cost - cumulative_cost = round(baseline_evaluator_cost + optimization.cost.total, 6) candidate_records: list[dict[str, Any]] = [] all_deltas = [] gate_decisions = [] all_results_comparable = True + candidate_evaluation_failed = False for index, candidate in enumerate(optimization.candidates, start=1): bundle = candidate_bundles[candidate.candidate_id] + canonical_candidate = CandidatePrompt( + candidate_id=candidate.candidate_id, + prompt=candidate.prompt, + rationale=candidate.rationale, + prompt_diff=candidate.prompt_diff, + prompt_fields=dict(bundle), + ) path_label = _candidate_artifact_component(index, candidate.candidate_id) + record: dict[str, Any] = { + "candidate": canonical_candidate, + "prompt_bundle": dict(bundle), + } _verify_snapshot_integrity(input_snapshots, "train") - train_result = await evaluator.evaluate( - prompt_id=candidate.candidate_id, - prompts=bundle, - dataset_path=train_path, - split="train", - trace=request.trace, - artifact_dir=_artifact_dir(run_root, "evaluations", path_label, "train"), + train_artifact_dir = _artifact_dir( + run_root, + "evaluations", + path_label, + "train", ) + try: + train_result = await evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=dict(bundle), + dataset_path=train_path, + split="train", + trace=request.trace, + artifact_dir=train_artifact_dir, + ) + except Exception as error: + record["evaluation_error"] = _recoverable_candidate_evaluation_failure( + error, + stage="train", + completed_results=[], + input_snapshots=input_snapshots, + prompt_snapshot=prompt_snapshot, + ) + candidate_records.append(record) + candidate_evaluation_failed = True + all_results_comparable = False + continue _verify_snapshot_integrity(input_snapshots, "train") _validate_eval_result( train_result, expected_prompt_id=candidate.candidate_id, expected_split="train", ) + record["train_result"] = train_result + explicit_evaluator_cost = round(explicit_evaluator_cost + train_result.cost, 6) _verify_snapshot_integrity(input_snapshots, "validation") - validation_result = await evaluator.evaluate( - prompt_id=candidate.candidate_id, - prompts=bundle, - dataset_path=validation_path, - split="validation", - trace=request.trace, - artifact_dir=_artifact_dir(run_root, "evaluations", path_label, "validation"), + validation_artifact_dir = _artifact_dir( + run_root, + "evaluations", + path_label, + "validation", ) + try: + validation_result = await evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=dict(bundle), + dataset_path=validation_path, + split="validation", + trace=request.trace, + artifact_dir=validation_artifact_dir, + ) + except Exception as error: + record["evaluation_error"] = _recoverable_candidate_evaluation_failure( + error, + stage="validation", + completed_results=[train_result], + input_snapshots=input_snapshots, + prompt_snapshot=prompt_snapshot, + ) + candidate_records.append(record) + candidate_evaluation_failed = True + all_results_comparable = False + continue _verify_snapshot_integrity(input_snapshots, "validation") _validate_eval_result( validation_result, expected_prompt_id=candidate.candidate_id, expected_split="validation", ) + record["validation_result"] = validation_result + explicit_evaluator_cost = round(explicit_evaluator_cost + validation_result.cost, 6) + candidate_records.append(record) + + run_cost_complete = optimization.cost.complete and not candidate_evaluation_failed + gate_cost_summary = replace(optimization.cost, complete=run_cost_complete) + cumulative_cost = round(baseline_evaluator_cost + optimization.cost.total, 6) + for record in candidate_records: + candidate = record["candidate"] + if "evaluation_error" in record: + decision = _evaluation_error_gate_decision( + candidate_id=candidate.candidate_id, + failure=record["evaluation_error"], + cumulative_cost=cumulative_cost, + ) + gate_decisions.append(decision) + cumulative_cost = decision.total_run_cost + continue + + train_result = record["train_result"] + validation_result = record["validation_result"] comparable = _result_pair_is_comparable(baseline_train, train_result) and ( _result_pair_is_comparable(baseline_validation, validation_result) ) @@ -310,22 +386,11 @@ async def _execute_with_snapshots( candidate_train=train_result, candidate_validation=validation_result, deltas=deltas, - cost_summary=optimization.cost, + cost_summary=gate_cost_summary, cumulative_cost=cumulative_cost, ) - candidate_records.append( - { - "candidate": candidate, - "train_result": train_result, - "validation_result": validation_result, - } - ) all_deltas.extend(deltas) gate_decisions.append(decision) - explicit_evaluator_cost = round( - explicit_evaluator_cost + train_result.cost + validation_result.cost, - 6, - ) cumulative_cost = decision.total_run_cost selected_candidate = _select_candidate(candidate_records, gate_decisions) @@ -334,7 +399,8 @@ async def _execute_with_snapshots( evaluator=round(optimization.cost.evaluator + explicit_evaluator_cost, 6), agent=optimization.cost.agent, total=cumulative_cost, - complete=optimization.cost.complete, + complete=run_cost_complete, + reported_optimizer_cost=_reported_optimizer_cost(optimization.cost), ) _validate_cost_summary(cost_summary, context="pipeline cost summary") input_hashes: dict[str, Any] = { @@ -436,7 +502,7 @@ async def _execute_with_snapshots( try: writeback = commit_prompt_bundle( prompt_snapshot, - candidate_bundles[selected_candidate], + dict(candidate_bundles[selected_candidate]), ) except ConcurrentPromptUpdateError as error: writeback = WritebackResult( @@ -687,7 +753,7 @@ def _public_input_drift( return public -def _prompt_bundle_hashes(bundle: dict[str, str]) -> dict[str, str]: +def _prompt_bundle_hashes(bundle: Mapping[str, str]) -> dict[str, str]: return {name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() for name, prompt in bundle.items()} @@ -780,12 +846,122 @@ def _decode_snapshot(snapshot: Any) -> dict[str, str]: return prompts +def _recoverable_candidate_evaluation_failure( + error: Exception, + *, + stage: str, + completed_results: list[EvalResult], + input_snapshots: _InputSnapshots, + prompt_snapshot: Any, +) -> dict[str, Any]: + """Return a public failure record only after proving prompt state is safe. + + Candidate backend errors are recoverable, but source-prompt restoration, + CAS, immutable-input, and cancellation failures remain run-fatal. + """ + + _verify_snapshot_integrity(input_snapshots) + if _exception_chain_contains( + error, + (ConcurrentPromptUpdateError, PromptRestorationError), + ): + raise error + + expected_prompt_hashes = prompt_snapshot.hashes() + observed_prompt_hashes = _current_prompt_hashes(prompt_snapshot) + if observed_prompt_hashes != expected_prompt_hashes: + changed = sorted( + name + for name in set(expected_prompt_hashes) | set(observed_prompt_hashes) + if expected_prompt_hashes.get(name) != observed_prompt_hashes.get(name) + ) + raise ConcurrentPromptUpdateError( + "source prompt integrity changed during failed candidate evaluation: " + + ", ".join(changed) + ) from error + + raw_message = str(error) + return { + "stage": stage, + "type": type(error).__name__, + "message": "candidate evaluation failed; backend details withheld", + "message_sha256": hashlib.sha256( + raw_message.encode("utf-8", errors="replace") + ).hexdigest(), + "completed_splits": [result.split for result in completed_results], + "known_evaluator_cost": round(sum(result.cost for result in completed_results), 6), + "cost_complete": False, + } + + +def _exception_chain_contains( + error: BaseException, + exception_types: tuple[type[BaseException], ...], +) -> bool: + pending: list[BaseException] = [error] + seen: set[int] = set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, exception_types): + return True + if current.__cause__ is not None: + pending.append(current.__cause__) + if current.__context__ is not None: + pending.append(current.__context__) + return False + + +def _evaluation_error_gate_decision( + *, + candidate_id: str, + failure: dict[str, Any], + cumulative_cost: float, +) -> GateDecision: + candidate_cost = round(float(failure["known_evaluator_cost"]), 6) + total_run_cost = round(cumulative_cost + candidate_cost, 6) + stage = str(failure["stage"]) + reason = ( + f"reject: evaluation_error during {stage}: " + f"{failure['type']}: {failure['message']}" + ) + return GateDecision( + candidate_id=candidate_id, + accepted=False, + reasons=[reason], + train_score_delta=None, + validation_score_delta=None, + new_hard_failures=[], + protected_regressions=[], + validation_new_failures=[], + excessive_score_drops=[], + overfit_detected=None, + candidate_cost=candidate_cost, + cumulative_cost=round(cumulative_cost, 6), + total_run_cost=total_run_cost, + cost=candidate_cost, + gate_status="not_applied", + gate_not_applied_reason="evaluation_error", + not_applied_checks=[ + "score_delta", + "validation_improvement", + "new_failures", + "hard_failures", + "protected_regressions", + "per_case_score_drop", + "overfit", + ], + ) + + def _validated_candidate_bundles( candidates: list[CandidatePrompt], *, expected_fields: set[str], -) -> dict[str, dict[str, str]]: - bundles: dict[str, dict[str, str]] = {} +) -> dict[str, Mapping[str, str]]: + bundles: dict[str, Mapping[str, str]] = {} casefold_ids: set[str] = set() for candidate in candidates: candidate_id = candidate.candidate_id @@ -823,7 +999,7 @@ def _validated_candidate_bundles( prompt_text, context=f"candidate {candidate_id!r} bundle field {field_name!r}", ) - bundles[candidate_id] = dict(bundle) + bundles[candidate_id] = MappingProxyType(dict(bundle)) return bundles @@ -980,6 +1156,12 @@ def _validate_cost_summary(summary: CostSummary, *, context: str) -> None: total = _finite_number(summary.total, context=f"{context} total", minimum=0.0) if not isinstance(summary.complete, bool): raise ValueError(f"{context} complete must be a boolean") + if summary.reported_optimizer_cost is not None: + _finite_number( + summary.reported_optimizer_cost, + context=f"{context} reported_optimizer_cost", + minimum=0.0, + ) if summary.complete and not math.isclose( total, sum(components.values()), @@ -989,6 +1171,14 @@ def _validate_cost_summary(summary: CostSummary, *, context: str) -> None: raise ValueError(f"{context} total must equal components when complete") +def _reported_optimizer_cost(summary: CostSummary) -> float | None: + if summary.complete: + return None + if summary.reported_optimizer_cost is not None: + return float(summary.reported_optimizer_cost) + return float(summary.total) + + def _validate_nested_numbers(value: Any, *, context: str) -> None: if isinstance(value, bool) or value is None or isinstance(value, str): return @@ -1058,7 +1248,12 @@ def _select_candidate( accepted: list[tuple[int, dict[str, Any]]] = [] for index, record in enumerate(candidate_records): candidate = record["candidate"] - if decisions_by_id[candidate.candidate_id].accepted: + decision = decisions_by_id[candidate.candidate_id] + if ( + decision.accepted + and "train_result" in record + and "validation_result" in record + ): accepted.append((index, record)) if not accepted: return None @@ -1113,7 +1308,7 @@ def _build_audit( baseline_train: EvalResult, baseline_validation: EvalResult, candidate_records: list[dict[str, Any]], - candidate_bundles: dict[str, dict[str, str]], + candidate_bundles: dict[str, Mapping[str, str]], optimization_raw_summary: dict[str, Any], optimization_cost: CostSummary, cost_summary: CostSummary, @@ -1123,15 +1318,35 @@ def _build_audit( ) -> dict[str, Any]: candidate_costs = { record["candidate"].candidate_id: round( - record["train_result"].cost + record["validation_result"].cost, + sum( + record[result_name].cost + for result_name in ("train_result", "validation_result") + if result_name in record + ), 6, ) for record in candidate_records } + candidate_evaluation_failures = { + record["candidate"].candidate_id: dict(record["evaluation_error"]) + for record in candidate_records + if "evaluation_error" in record + } candidate_prompt_hashes = { candidate_id: {name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() for name, prompt in bundle.items()} for candidate_id, bundle in candidate_bundles.items() } + cost_audit: dict[str, Any] = { + "baseline": round(baseline_train.cost + baseline_validation.cost, 6), + "candidates": candidate_costs, + "optimization": to_jsonable(optimization_cost), + "evaluator": cost_summary.evaluator, + "total": cost_summary.total if cost_summary.complete else None, + "complete": cost_summary.complete, + "reported_optimizer_cost": cost_summary.reported_optimizer_cost, + } + if not cost_summary.complete: + cost_audit["known_run_cost"] = cost_summary.total duration = max(time.perf_counter() - started, 1e-9) target_paths = {name: _display_path(path) for name, path in request.target_prompt_paths.items()} return { @@ -1163,23 +1378,24 @@ def _build_audit( ) for index, record in enumerate(candidate_records, start=1) }, - "candidate_prompts": candidate_bundles, + "candidate_prompts": { + candidate_id: dict(bundle) for candidate_id, bundle in candidate_bundles.items() + }, + "candidate_evaluation_failures": candidate_evaluation_failures, "prompt_diffs": { record["candidate"].candidate_id: record["candidate"].prompt_diff for record in candidate_records }, - "total_run_cost": cost_summary.total, - "cost": { - "baseline": round(baseline_train.cost + baseline_validation.cost, 6), - "candidates": candidate_costs, - "optimization": to_jsonable(optimization_cost), - "evaluator": cost_summary.evaluator, - "total": cost_summary.total, - "complete": cost_summary.complete, - }, + "total_run_cost": cost_summary.total if cost_summary.complete else None, + "known_run_cost": cost_summary.total if not cost_summary.complete else None, + "total_run_cost_complete": cost_summary.complete, + "cost": cost_audit, "sdk_result_summary": _sanitize_public_value(to_jsonable(optimization_raw_summary)), "sdk_result_availability": { "aggregate_validation_result": bool(request.mode == "sdk" and optimization_raw_summary), - "full_train_eval_result": True, + "full_train_eval_result": not any( + failure.get("stage") == "train" + for failure in candidate_evaluation_failures.values() + ), "full_per_case_validation_delta": all_results_comparable, }, "reproducibility_shell": "powershell", diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index c44bc717..c645c892 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -105,8 +105,10 @@ def build_report( ) -> OptimizationReport: all_results: list[EvalResult] = [baseline_train, baseline_validation] for record in candidate_records: - all_results.append(record["train_result"]) - all_results.append(record["validation_result"]) + for result_name in ("train_result", "validation_result"): + result = record.get(result_name) + if isinstance(result, EvalResult): + all_results.append(result) return OptimizationReport( schema_version="eval_optimize_loop.v1", run=run, @@ -313,12 +315,20 @@ def render_markdown(report: OptimizationReport) -> str: "", ]) availability = report.audit.get("sdk_result_availability", {}) - lines.extend([ - "Fake and SDK modes perform complete AgentEvaluator-compatible reevaluation for baseline " - "and every candidate on both train and validation; optimizer aggregates are never used " - "as gate evidence.", - "", - ]) + if report.audit.get("candidate_evaluation_failures"): + lines.extend([ + "Fake and SDK modes require complete AgentEvaluator-compatible train and validation " + "reevaluation before gating a candidate; evaluation errors are recorded as explicit " + "rejections and optimizer aggregates are never used as gate evidence.", + "", + ]) + else: + lines.extend([ + "Fake and SDK modes perform complete AgentEvaluator-compatible reevaluation for baseline " + "and every candidate on both train and validation; optimizer aggregates are never used " + "as gate evidence.", + "", + ]) if report.run.get("mode") == "sdk": lines.extend([ "SDK availability: " @@ -350,9 +360,17 @@ def render_markdown(report: OptimizationReport) -> str: candidate: CandidatePrompt = record["candidate"] gate = decision_by_id[candidate.candidate_id] verdict = "accept" if gate.accepted else "reject" + train_result = record.get("train_result") + validation_result = record.get("validation_result") + train_score = f"{train_result.score:.3f}" if isinstance(train_result, EvalResult) else "n/a" + validation_score = ( + f"{validation_result.score:.3f}" + if isinstance(validation_result, EvalResult) + else "n/a" + ) lines.append( - f"| {candidate.candidate_id} | {record['train_result'].score:.3f} | " - f"{record['validation_result'].score:.3f} | {verdict} |" + f"| {candidate.candidate_id} | {train_score} | " + f"{validation_score} | {verdict} |" ) lines.extend([ @@ -390,11 +408,24 @@ def render_markdown(report: OptimizationReport) -> str: lines.append("") lines.append(f"Attribution accuracy: {summary['attribution_accuracy']:.3f}") + cost_audit = report.audit.get("cost", {}) + lines.extend(["", "## Cost And Audit", ""]) + if cost_audit.get("complete"): + lines.append(f"Total cost: {cost_audit.get('total', 0):.3f}") + else: + reported_optimizer_cost = cost_audit.get("reported_optimizer_cost") + if reported_optimizer_cost is not None: + lines.append( + "Reported optimizer cost (incomplete; not total run cost): " + f"{reported_optimizer_cost:.3f}" + ) + lines.append(f"Known evaluator cost: {cost_audit.get('evaluator', 0):.3f}") + lines.append( + "Known run cost (incomplete; not total run cost): " + f"{cost_audit.get('known_run_cost', 0):.3f}" + ) + lines.append("Complete run cost: unavailable") lines.extend([ - "", - "## Cost And Audit", - "", - f"Total cost: {report.audit.get('cost', {}).get('total', 0):.3f}", f"Config hash: `{report.audit.get('config_hash', '')}`", f"Run id: `{report.run.get('run_id', '')}`", ]) @@ -456,12 +487,17 @@ def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None candidate_name = _candidate_artifact_name(index, candidate.candidate_id) candidate_dir = prompt_dir / candidate_name candidate_dir.mkdir(exist_ok=True) - for field_name, prompt_text in candidate.bundle().items(): + prompt_bundle = record.get("prompt_bundle") + if prompt_bundle is None: + prompt_bundle = candidate.bundle() + for field_name, prompt_text in prompt_bundle.items(): field_artifact = _safe_artifact_name(str(field_name)) _atomic_write_text(candidate_dir / f"{field_artifact}.txt", prompt_text) _atomic_write_text(diffs_dir / f"{candidate_name}.diff", candidate.prompt_diff) for split_name in ("train_result", "validation_result"): - split_result = record[split_name] + split_result = record.get(split_name) + if not isinstance(split_result, EvalResult): + continue split_artifact = _safe_artifact_name(str(split_result.split)) path = results_dir / f"{candidate_name}_{split_artifact}.json" _atomic_write_text(path, _json_text(split_result)) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py index 04da133e..9741ad47 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -110,6 +110,7 @@ class CostSummary: agent: float = 0.0 total: float = 0.0 complete: bool = True + reported_optimizer_cost: float | None = None @dataclass(frozen=True) @@ -177,13 +178,13 @@ class GateDecision: candidate_id: str accepted: bool reasons: list[str] - train_score_delta: float - validation_score_delta: float + train_score_delta: float | None + validation_score_delta: float | None new_hard_failures: list[str] protected_regressions: list[str] validation_new_failures: list[str] excessive_score_drops: list[str] - overfit_detected: bool + overfit_detected: bool | None candidate_cost: float cumulative_cost: float total_run_cost: float diff --git a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py index 3a93c940..4df6202d 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py @@ -22,6 +22,10 @@ class ConcurrentPromptUpdateError(RuntimeError): """Raised when source prompts no longer match their captured snapshot.""" +class PromptRestorationError(RuntimeError): + """Raised when temporarily installed prompts cannot be restored safely.""" + + @dataclass(frozen=True) class PromptFileSnapshot: """Byte-for-byte snapshot of one source prompt file.""" @@ -209,17 +213,19 @@ def temporary_prompt_bundle( restoration_error = _restoration_error(snapshot, failures) if restoration_error is not None: diagnostic = f"failed to restore prompt snapshot: {restoration_error}" + restore_error = PromptRestorationError(diagnostic) add_note = getattr(primary_error, "add_note", None) if add_note is not None: add_note(diagnostic) - else: - raise primary_error from RuntimeError(diagnostic) + raise primary_error from restore_error raise else: failures = _restore_snapshot(snapshot, written_hashes) restoration_error = _restoration_error(snapshot, failures) if restoration_error is not None: - raise RuntimeError(f"failed to restore prompt snapshot: {restoration_error}") + raise PromptRestorationError( + f"failed to restore prompt snapshot: {restoration_error}" + ) def commit_prompt_bundle( diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py index 04b3fe3a..d958f4f4 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py @@ -23,6 +23,7 @@ from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationRound from examples.optimization.eval_optimize_loop.eval_loop.schemas import WritebackResult from examples.optimization.eval_optimize_loop.eval_loop.writeback import ConcurrentPromptUpdateError +from examples.optimization.eval_optimize_loop.eval_loop.writeback import PromptRestorationError from examples.optimization.eval_optimize_loop.run_pipeline import build_pipeline_request_and_backend @@ -78,6 +79,67 @@ async def optimize_candidates( ) +class MutatingBundleBackend(RecordingBackend): + async def evaluate(self, **kwargs: Any) -> EvalResult: + result = await super().evaluate(**kwargs) + kwargs["prompts"]["system_prompt"] = "backend-mutated prompt" + return result + + async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: + result = await super().optimize_candidates(**kwargs) + kwargs["baseline_prompts"]["system_prompt"] = "optimizer-mutated baseline" + return result + + +class FailingCandidateBackend(RecordingBackend): + def __init__( + self, + *, + fail_on: tuple[str, str], + failure: Exception, + delegate: RecordingBackend, + ) -> None: + super().__init__( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + self.fail_on = fail_on + self.failure = failure + + async def evaluate(self, **kwargs: Any) -> EvalResult: + if (kwargs["prompt_id"], kwargs["split"]) == self.fail_on: + self.calls.append( + ( + "evaluate", + kwargs["prompt_id"], + kwargs["split"], + dict(kwargs["prompts"]), + Path(kwargs["artifact_dir"]), + kwargs["trace"], + ) + ) + raise self.failure + return await super().evaluate(**kwargs) + + +class AllCandidatesFailingBackend(RecordingBackend): + async def evaluate(self, **kwargs: Any) -> EvalResult: + if kwargs["prompt_id"] != "baseline": + self.calls.append( + ( + "evaluate", + kwargs["prompt_id"], + kwargs["split"], + dict(kwargs["prompts"]), + Path(kwargs["artifact_dir"]), + kwargs["trace"], + ) + ) + raise RuntimeError(f"{kwargs['prompt_id']} evaluation unavailable") + return await super().evaluate(**kwargs) + + @pytest.mark.asyncio async def test_execute_pipeline_uses_train_only_evidence_and_full_candidate_reevaluation(tmp_path: Path): request = _request(tmp_path, update_source=False) @@ -108,6 +170,265 @@ async def test_execute_pipeline_uses_train_only_evidence_and_full_candidate_reev assert report.audit["sdk_result_availability"]["full_per_case_validation_delta"] is True +@pytest.mark.asyncio +async def test_backend_cannot_mutate_canonical_baseline_or_candidate_bundles(tmp_path: Path): + request = _request(tmp_path, update_source=True) + delegate = _safe_backend(candidate_count=1) + backend = MutatingBundleBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + prompt_calls = [ + (call[1], call[2], call[3]["system_prompt"]) + for call in backend.calls + if call[0] == "evaluate" + ] + assert prompt_calls == [ + ("baseline", "train", "baseline\n"), + ("baseline", "validation", "baseline\n"), + ("candidate_a", "train", "safe prompt"), + ("candidate_a", "validation", "safe prompt"), + ] + optimize_call = next(call for call in backend.calls if call[0] == "optimize") + assert optimize_call[1] == {"system_prompt": "baseline\n"} + assert report.candidates[0]["candidate"].bundle() == {"system_prompt": "safe prompt"} + assert report.candidates[0]["prompt_bundle"] == {"system_prompt": "safe prompt"} + assert report.audit["candidate_prompts"]["candidate_a"] == { + "system_prompt": "safe prompt" + } + assert report.audit["candidate_prompt_hashes"]["candidate_a"] == { + "system_prompt": hashlib.sha256(b"safe prompt").hexdigest() + } + candidate_artifact = report.audit["candidate_artifacts"]["candidate_a"] + artifact_prompt = ( + Path(request.output_dir) + / "runs" + / request.run_id + / "candidate_prompts" + / candidate_artifact + / "system_prompt.txt" + ) + assert artifact_prompt.read_text(encoding="utf-8") == "safe prompt" + assert Path(request.target_prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "safe prompt" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_split", ["train", "validation"]) +async def test_candidate_evaluation_error_rejects_candidate_and_continues( + tmp_path: Path, + failed_split: str, +): + request = _request(tmp_path, update_source=True) + backend = FailingCandidateBackend( + fail_on=("candidate_a", failed_split), + failure=RuntimeError(f"synthetic {failed_split} evaluation failure"), + delegate=_safe_backend(candidate_count=2), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + candidate_calls = [ + (call[1], call[2]) for call in backend.calls if call[0] == "evaluate" and call[1] != "baseline" + ] + expected_a_calls = [("candidate_a", "train")] + if failed_split == "validation": + expected_a_calls.append(("candidate_a", "validation")) + assert candidate_calls == expected_a_calls + [ + ("candidate_b", "train"), + ("candidate_b", "validation"), + ] + assert report.selected_candidate == "candidate_b" + assert report.writeback.status == "applied" + assert Path(request.target_prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "other prompt" + + failed_record = report.candidates[0] + assert failed_record["candidate"].candidate_id == "candidate_a" + assert failed_record["evaluation_error"]["stage"] == failed_split + assert failed_record["evaluation_error"]["type"] == "RuntimeError" + assert failed_record["evaluation_error"]["cost_complete"] is False + assert ("train_result" in failed_record) is (failed_split == "validation") + assert "validation_result" not in failed_record + + failed_decision = report.gate_decisions[0] + assert failed_decision.accepted is False + assert failed_decision.gate_status == "not_applied" + assert failed_decision.gate_not_applied_reason == "evaluation_error" + assert failed_decision.train_score_delta is None + assert failed_decision.validation_score_delta is None + assert any("evaluation_error" in reason and failed_split in reason for reason in failed_decision.reasons) + assert failed_decision.not_applied_checks + + audit_failure = report.audit["candidate_evaluation_failures"]["candidate_a"] + assert audit_failure == failed_record["evaluation_error"] + assert report.cost_summary.complete is False + expected_known_cost = 0.05 if failed_split == "validation" else 0.04 + assert report.cost_summary.total == expected_known_cost + assert report.audit["cost"]["total"] is None + assert report.audit["cost"]["known_run_cost"] == expected_known_cost + markdown = report_module.render_markdown(report) + assert "evaluation_error" in markdown + assert "n/a" in markdown + + +@pytest.mark.asyncio +async def test_candidate_evaluation_error_makes_later_budget_gate_fail_closed( + tmp_path: Path, +): + request = _request( + tmp_path, + update_source=True, + gate_config={ + "min_val_score_improvement": 0.0, + "max_score_drop_per_case": 1.0, + "max_total_cost": 10.0, + }, + ) + backend = FailingCandidateBackend( + fail_on=("candidate_a", "train"), + failure=RuntimeError("unknown failed evaluation cost"), + delegate=_safe_backend(candidate_count=2), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.writeback.status == "rejected" + assert report.gate_decisions[0].gate_not_applied_reason == "evaluation_error" + assert any("cost_unavailable" in reason for reason in report.gate_decisions[1].reasons) + assert report.cost_summary.complete is False + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + + +@pytest.mark.asyncio +async def test_all_candidate_evaluation_errors_finalize_rejected_audit_without_writeback( + tmp_path: Path, +): + request = _request(tmp_path, update_source=True) + delegate = _safe_backend(candidate_count=2) + backend = AllCandidatesFailingBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.writeback.status == "rejected" + assert report.audit["writeback_journal"]["state"] == "rejected" + assert report.audit["writeback_journal"]["report_phase"] == "final" + assert set(report.audit["candidate_evaluation_failures"]) == { + "candidate_a", + "candidate_b", + } + assert all("evaluation_error" in record for record in report.candidates) + assert all("train_result" not in record for record in report.candidates) + assert all(decision.gate_status == "not_applied" for decision in report.gate_decisions) + assert report.cost_summary.complete is False + assert report.cost_summary.total == 0.02 + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + run_dir = Path(request.output_dir) / "runs" / request.run_id + assert (run_dir / "optimization_report.json").is_file() + assert (run_dir / "audit.json").is_file() + assert (run_dir / "writeback.json").is_file() + + +@pytest.mark.asyncio +async def test_candidate_evaluation_error_does_not_persist_backend_secret_text( + tmp_path: Path, +): + request = _request(tmp_path) + secret = "api_key=sk-private-value token=github-private-value" + backend = FailingCandidateBackend( + fail_on=("candidate_a", "train"), + failure=RuntimeError(secret), + delegate=_safe_backend(candidate_count=2), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + failure = report.candidates[0]["evaluation_error"] + assert failure["message"] == "candidate evaluation failed; backend details withheld" + assert failure["message_sha256"] == hashlib.sha256(secret.encode("utf-8")).hexdigest() + serialized = report_module.report_to_json(report) + report_module.render_markdown(report) + serialized += (Path(request.output_dir) / "runs" / request.run_id / "audit.json").read_text( + encoding="utf-8" + ) + assert "sk-private-value" not in serialized + assert "github-private-value" not in serialized + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_kind", ["cas", "direct_restore", "chained_restore"]) +async def test_candidate_prompt_integrity_failures_abort_run( + tmp_path: Path, + failure_kind: str, +): + request = _request(tmp_path, update_source=True) + if failure_kind == "cas": + failure: Exception = ConcurrentPromptUpdateError("source changed") + elif failure_kind == "direct_restore": + failure = PromptRestorationError("restore failed") + else: + failure = RuntimeError("candidate evaluation failed") + failure.__cause__ = PromptRestorationError("restore failed") + backend = FailingCandidateBackend( + fail_on=("candidate_a", "train"), + failure=failure, + delegate=_safe_backend(candidate_count=2), + ) + + with pytest.raises(Exception) as error_info: + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert error_info.value is failure + assert not any(call[0] == "evaluate" and call[1] == "candidate_b" for call in backend.calls) + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("mutation_target", "error_pattern"), + [ + ("prompt", "source prompt integrity changed"), + ("input", "immutable input snapshot.*train"), + ], +) +async def test_candidate_failure_with_state_drift_aborts_run( + tmp_path: Path, + mutation_target: str, + error_pattern: str, +): + request = _request(tmp_path, update_source=True) + + class StateDriftBackend(FailingCandidateBackend): + async def evaluate(self, **kwargs: Any) -> EvalResult: + if (kwargs["prompt_id"], kwargs["split"]) == self.fail_on: + if mutation_target == "prompt": + Path(request.target_prompt_paths["system_prompt"]).write_text( + "external prompt update", + encoding="utf-8", + ) + else: + Path(kwargs["dataset_path"]).write_text("{}", encoding="utf-8") + return await super().evaluate(**kwargs) + + backend = StateDriftBackend( + fail_on=("candidate_a", "train"), + failure=RuntimeError("ordinary evaluation failure"), + delegate=_safe_backend(candidate_count=2), + ) + + with pytest.raises((ConcurrentPromptUpdateError, ValueError), match=error_pattern): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert not any(call[0] == "evaluate" and call[1] == "candidate_b" for call in backend.calls) + + @pytest.mark.asyncio async def test_rejected_candidate_never_changes_source_when_writeback_requested(tmp_path: Path): request = _request( @@ -264,6 +585,15 @@ async def test_incomplete_optimizer_cost_is_fail_closed_only_when_budget_configu assert any(reason_fragment in reason for reason in report.gate_decisions[0].reasons) else: assert all("cost_unavailable" not in reason for reason in report.gate_decisions[0].reasons) + payload = json.loads(report_module.report_to_json(report)) + assert payload["cost_summary"]["complete"] is False + assert payload["cost_summary"]["reported_optimizer_cost"] == 0.2 + assert report.audit["cost"]["complete"] is False + assert report.audit["cost"]["total"] is None + assert report.audit["cost"]["reported_optimizer_cost"] == 0.2 + markdown = report_module.render_markdown(report) + assert "Reported optimizer cost (incomplete; not total run cost): 0.200" in markdown + assert "Total cost:" not in markdown @pytest.mark.asyncio @@ -287,6 +617,14 @@ async def test_optimizer_cost_is_counted_once_across_candidates(tmp_path: Path): total=0.78, complete=True, ) + assert report.audit["total_run_cost"] == 0.78 + assert report.audit["known_run_cost"] is None + assert report.audit["total_run_cost_complete"] is True + assert report.audit["cost"]["total"] == 0.78 + assert report.audit["cost"]["reported_optimizer_cost"] is None + markdown = report_module.render_markdown(report) + assert "Total cost: 0.780" in markdown + assert "Reported optimizer cost" not in markdown @pytest.mark.asyncio diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 01c402df..d60c32e9 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -214,6 +214,7 @@ async def test_sdk_backend_maps_rounds_deduplicates_bundles_and_marks_cost_incom assert result.rounds[1].cost.total == 0.2 assert result.cost.total == 0.3 assert result.cost.complete is False + assert result.cost.reported_optimizer_cost == 0.3 assert result.raw_summary["rounds"][1]["acceptance_reason"] == "duplicate proposal" assert result.raw_summary["rounds"][1]["accepted"] is True @@ -1207,7 +1208,7 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo ) output_dir = tmp_path / "sdk_run" - payload = (output_dir / "optimization_report.json").read_text(encoding="utf-8") + payload = json.loads((output_dir / "optimization_report.json").read_text(encoding="utf-8")) markdown = (output_dir / "optimization_report.md").read_text(encoding="utf-8") assert report.run["mode"] == "sdk" assert report.run["update_source"] is False @@ -1219,8 +1220,16 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo assert report.gate_decisions[0].gate_status == "applied" assert report.gate_decisions[0].not_applied_checks == [] assert report.audit["duration_seconds"] > 0 - assert report.audit["total_run_cost"] == 0.123 - assert report.audit["cost"]["total"] == 0.123 + assert report.audit["total_run_cost"] is None + assert report.audit["known_run_cost"] == 0.123 + assert report.audit["total_run_cost_complete"] is False + assert report.audit["cost"]["total"] is None + assert report.audit["cost"]["complete"] is False + assert report.audit["cost"]["reported_optimizer_cost"] == 0.123 + assert payload["cost_summary"]["reported_optimizer_cost"] == 0.123 + assert payload["cost_summary"]["complete"] is False + assert "Reported optimizer cost (incomplete; not total run cost): 0.123" in markdown + assert "Total cost:" not in markdown assert report.audit["sdk_result_summary"]["status"] == "SUCCEEDED" assert report.audit["sdk_result_summary"]["baseline_metric_breakdown"] == {"exact_match": 0.5} assert report.audit["sdk_result_summary"]["best_metric_breakdown"] == {"exact_match": 0.75} diff --git a/examples/optimization/eval_optimize_loop/tests/test_writeback.py b/examples/optimization/eval_optimize_loop/tests/test_writeback.py index 6e83a097..7d2fbb30 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_writeback.py +++ b/examples/optimization/eval_optimize_loop/tests/test_writeback.py @@ -10,6 +10,7 @@ from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( ConcurrentPromptUpdateError, ) +from examples.optimization.eval_optimize_loop.eval_loop.writeback import PromptRestorationError from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( commit_prompt_bundle, ) @@ -110,7 +111,7 @@ def test_temporary_restore_preserves_external_update_and_reports_conflict(tmp_pa snapshot = snapshot_prompt_files(paths) external_content = b"external during temporary evaluation" - with pytest.raises(RuntimeError, match="restore"): + with pytest.raises(PromptRestorationError, match="restore"): with temporary_prompt_bundle( snapshot, {"system": "candidate system", "user": "candidate user"}, @@ -142,6 +143,7 @@ def test_temporary_body_error_remains_primary_when_restore_conflicts(tmp_path: P assert "restore" in diagnostics assert "user" in diagnostics + assert isinstance(error_info.value.__cause__, PromptRestorationError) assert paths["system"].read_bytes() == baseline["system"] assert paths["user"].read_bytes() == external_content From 35a99bf389a92779f2afce747250c348f825735d Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 03:30:37 +0800 Subject: [PATCH 27/34] fix(examples): isolate physical inputs and prompt state --- .../eval_optimize_loop/eval_loop/artifacts.py | 66 +++ .../eval_optimize_loop/eval_loop/backends.py | 11 +- .../eval_optimize_loop/eval_loop/config.py | 9 +- .../eval_optimize_loop/eval_loop/pipeline.py | 318 ++++++++++++--- .../eval_optimize_loop/eval_loop/report.py | 16 +- .../eval_optimize_loop/eval_loop/writeback.py | 2 + .../eval_optimize_loop/run_pipeline.py | 21 +- .../tests/test_pipeline_orchestration.py | 385 +++++++++++++++++- .../tests/test_sdk_backend.py | 20 +- .../tests/test_writeback.py | 11 + 10 files changed, 777 insertions(+), 82 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py b/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py index 901e53cc..8a08b4f0 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py @@ -3,6 +3,8 @@ from __future__ import annotations import re +from collections.abc import Mapping +from pathlib import Path from typing import Any @@ -33,3 +35,67 @@ def validate_artifact_component(value: Any, *, context: str) -> str: if windows_stem in WINDOWS_RESERVED_NAMES: raise ValueError(f"unsafe {context}: {value!r} is reserved on Windows") return value + + +def validate_distinct_file_paths( + paths: Mapping[str, str | Path], + *, + context: str, +) -> None: + """Reject aliases using portable case-folding and physical file identity.""" + + resolved_keys: dict[str, str] = {} + physical_keys: dict[tuple[int, int], str] = {} + observed_paths: list[tuple[str, Path, bool]] = [] + for label, raw_path in paths.items(): + path = Path(raw_path) + try: + resolved = path.resolve(strict=False) + except OSError as error: + raise ValueError(f"{context} {label!r} cannot be resolved: {error}") from error + + portable_key = str(resolved).casefold() + previous_label = resolved_keys.get(portable_key) + if previous_label is not None: + raise ValueError( + f"{context} must be different physical files; " + f"{previous_label!r} and {label!r} collide case-insensitively" + ) + + try: + metadata = path.stat() + except (FileNotFoundError, NotADirectoryError): + metadata = None + except OSError as error: + raise ValueError(f"{context} {label!r} is unavailable: {error}") from error + + physical_key = ( + (int(metadata.st_dev), int(metadata.st_ino)) + if metadata is not None and int(metadata.st_ino) != 0 + else None + ) + if physical_key is not None and physical_key in physical_keys: + raise ValueError( + f"{context} must be different physical files; " + f"{physical_keys[physical_key]!r} and {label!r} are aliases" + ) + + for observed_label, observed_path, observed_exists in observed_paths: + if metadata is None or not observed_exists: + continue + try: + same_file = path.samefile(observed_path) + except (FileNotFoundError, NotADirectoryError): + same_file = False + except OSError as error: + raise ValueError(f"{context} {label!r} is unavailable: {error}") from error + if same_file: + raise ValueError( + f"{context} must be different physical files; " + f"{observed_label!r} and {label!r} are aliases" + ) + + resolved_keys[portable_key] = label + if physical_key is not None: + physical_keys[physical_key] = label + observed_paths.append((label, path, metadata is not None)) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 414b9606..0c0c3754 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -13,6 +13,7 @@ from typing import Protocol from .diffing import make_unified_diff +from .artifacts import validate_distinct_file_paths from .evaluator import ExampleEvaluator from .fake_judge import FakeJudge from .fake_model import FakeModel @@ -532,9 +533,13 @@ def _load_required_call_agent(self, *, for_evaluation: bool): return _load_call_agent(self.call_agent_path) def _target_prompt_paths(self) -> dict[str, str | Path]: - if self.target_prompt_paths: - return dict(self.target_prompt_paths) - return {"system_prompt": self.prompt_path} + paths = ( + dict(self.target_prompt_paths) + if self.target_prompt_paths + else {"system_prompt": self.prompt_path} + ) + validate_distinct_file_paths(paths, context="SDK target prompt fields") + return paths def _required_system_prompt(prompts: dict[str, str], *, context: str) -> str: diff --git a/examples/optimization/eval_optimize_loop/eval_loop/config.py b/examples/optimization/eval_optimize_loop/eval_loop/config.py index 8f092702..5e80865f 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/config.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/config.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any +from .artifacts import validate_distinct_file_paths from .schemas import EvalCase @@ -141,10 +142,10 @@ def validate_inputs( validation_cases: list[EvalCase], config: OptimizerConfig, ) -> None: - train_resolved = Path(train_path).resolve() - val_resolved = Path(val_path).resolve() - if train_resolved == val_resolved: - raise ValueError(f"{train_path}: train and validation evalset paths must be different") + validate_distinct_file_paths( + {"train": train_path, "validation": val_path}, + context="train and validation evalset paths", + ) _validate_cases(train_cases, split="train", path=train_path) _validate_cases(validation_cases, split="validation", path=val_path) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py index 114979af..a3f9a1b2 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py @@ -9,6 +9,8 @@ import re import tempfile import time +from collections.abc import Awaitable +from collections.abc import Callable from collections.abc import Mapping from dataclasses import dataclass from dataclasses import replace @@ -18,6 +20,7 @@ from .attribution import summarize_failures from .artifacts import validate_artifact_component +from .artifacts import validate_distinct_file_paths from .backends import EvaluationBackend from .backends import OptimizationBackend from .config import _parse_gate_config @@ -112,8 +115,10 @@ async def execute_pipeline( if not request.target_prompt_paths: raise ValueError("target_prompt_paths must not be empty") _validate_target_prompt_fields(request.target_prompt_paths) - if Path(request.train_path).resolve() == Path(request.validation_path).resolve(): - raise ValueError("train and validation evalset paths must be different") + validate_distinct_file_paths( + {"train": request.train_path, "validation": request.validation_path}, + context="train and validation evalset paths", + ) run_root = _claim_run_directory(request.output_dir, run_id=run_id) input_snapshots = _snapshot_pipeline_inputs(request, run_id=run_id) @@ -142,7 +147,6 @@ async def execute_pipeline( def _validate_target_prompt_fields(target_prompt_paths: dict[str, str | Path]) -> None: seen: set[str] = set() - resolved_paths: set[Path] = set() for field_name, prompt_path in target_prompt_paths.items(): try: validate_artifact_component(field_name, context="target prompt field") @@ -154,10 +158,10 @@ def _validate_target_prompt_fields(target_prompt_paths: dict[str, str | Path]) - raise ValueError("target prompt field names must be case-insensitively unique") seen.add(normalized_name) - resolved_path = Path(prompt_path).resolve() - if resolved_path in resolved_paths: - raise ValueError("target prompt fields must not reference the same resolved file") - resolved_paths.add(resolved_path) + validate_distinct_file_paths( + target_prompt_paths, + context="target prompt fields", + ) async def _execute_with_snapshots( @@ -200,13 +204,17 @@ async def _execute_with_snapshots( _verify_snapshot_integrity(input_snapshots) _verify_snapshot_integrity(input_snapshots, "train") - baseline_train = await evaluator.evaluate( - prompt_id="baseline", - prompts=dict(baseline_prompts), - dataset_path=train_path, - split="train", - trace=request.trace, - artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "train"), + baseline_train = await _await_backend_with_prompt_integrity( + lambda: evaluator.evaluate( + prompt_id="baseline", + prompts=dict(baseline_prompts), + dataset_path=train_path, + split="train", + trace=request.trace, + artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "train"), + ), + prompt_snapshot=prompt_snapshot, + context="baseline train evaluation", ) _verify_snapshot_integrity(input_snapshots, "train") _validate_eval_result( @@ -216,13 +224,17 @@ async def _execute_with_snapshots( expected_case_ids=set(train_metadata), ) _verify_snapshot_integrity(input_snapshots, "validation") - baseline_validation = await evaluator.evaluate( - prompt_id="baseline", - prompts=dict(baseline_prompts), - dataset_path=validation_path, - split="validation", - trace=request.trace, - artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "validation"), + baseline_validation = await _await_backend_with_prompt_integrity( + lambda: evaluator.evaluate( + prompt_id="baseline", + prompts=dict(baseline_prompts), + dataset_path=validation_path, + split="validation", + trace=request.trace, + artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "validation"), + ), + prompt_snapshot=prompt_snapshot, + context="baseline validation evaluation", ) _verify_snapshot_integrity(input_snapshots, "validation") _validate_eval_result( @@ -234,14 +246,18 @@ async def _execute_with_snapshots( failure_summary = summarize_failures([baseline_train]) _verify_snapshot_integrity(input_snapshots, "train", "validation", "optimizer") - optimization = await optimizer.optimize_candidates( - baseline_prompts=dict(baseline_prompts), - baseline_train=baseline_train, - failure_summary=failure_summary, - train_path=train_path, - validation_path=validation_path, - config_path=optimizer_path, - artifact_dir=_artifact_dir(run_root, "optimizer"), + optimization = await _await_backend_with_prompt_integrity( + lambda: optimizer.optimize_candidates( + baseline_prompts=dict(baseline_prompts), + baseline_train=baseline_train, + failure_summary=failure_summary, + train_path=train_path, + validation_path=validation_path, + config_path=optimizer_path, + artifact_dir=_artifact_dir(run_root, "optimizer"), + ), + prompt_snapshot=prompt_snapshot, + context="candidate optimization", ) _verify_snapshot_integrity(input_snapshots, "train", "validation", "optimizer") _validate_optimization_result(optimization) @@ -281,13 +297,17 @@ async def _execute_with_snapshots( "train", ) try: - train_result = await evaluator.evaluate( - prompt_id=candidate.candidate_id, - prompts=dict(bundle), - dataset_path=train_path, - split="train", - trace=request.trace, - artifact_dir=train_artifact_dir, + train_result = await _await_backend_with_prompt_integrity( + lambda: evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=dict(bundle), + dataset_path=train_path, + split="train", + trace=request.trace, + artifact_dir=train_artifact_dir, + ), + prompt_snapshot=prompt_snapshot, + context=f"candidate {candidate.candidate_id!r} train evaluation", ) except Exception as error: record["evaluation_error"] = _recoverable_candidate_evaluation_failure( @@ -317,13 +337,17 @@ async def _execute_with_snapshots( "validation", ) try: - validation_result = await evaluator.evaluate( - prompt_id=candidate.candidate_id, - prompts=dict(bundle), - dataset_path=validation_path, - split="validation", - trace=request.trace, - artifact_dir=validation_artifact_dir, + validation_result = await _await_backend_with_prompt_integrity( + lambda: evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=dict(bundle), + dataset_path=validation_path, + split="validation", + trace=request.trace, + artifact_dir=validation_artifact_dir, + ), + prompt_snapshot=prompt_snapshot, + context=f"candidate {candidate.candidate_id!r} validation evaluation", ) except Exception as error: record["evaluation_error"] = _recoverable_candidate_evaluation_failure( @@ -464,6 +488,10 @@ async def _execute_with_snapshots( writeback=provisional_writeback, ) + _verify_prompt_integrity( + prompt_snapshot, + context="immediately before preparing run artifacts", + ) artifact_paths = prepare_run_artifacts(report, request.output_dir) if selected_candidate is None or not request.update_source: terminal_journal = _terminal_journal( @@ -472,8 +500,32 @@ async def _execute_with_snapshots( state=provisional_writeback.status, ) final_report = _with_writeback(report, terminal_journal, provisional_writeback) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="before persisting no-write outcome", + artifact_paths=artifact_paths, + journal=terminal_journal, + request=request, + ) persist_writeback_outcome(final_report, artifact_paths) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="immediately before finalizing no-write run", + artifact_paths=artifact_paths, + journal=terminal_journal, + request=request, + ) finalize_run_artifacts(final_report, artifact_paths) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="immediately before returning no-write run", + artifact_paths=artifact_paths, + journal=terminal_journal, + request=request, + ) return final_report input_drift = _public_input_drift(_input_drift(input_snapshots), request) @@ -492,8 +544,32 @@ async def _execute_with_snapshots( input_drift=input_drift, ) final_report = _with_writeback(report, final_journal, writeback, input_drift=input_drift) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="before persisting input-drift outcome", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ) persist_writeback_outcome(final_report, artifact_paths) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="immediately before finalizing input-drift run", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ) finalize_run_artifacts(final_report, artifact_paths) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="immediately before returning input-drift run", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ) return final_report committing_journal = dict(writeback_journal) @@ -518,6 +594,15 @@ async def _execute_with_snapshots( else: writeback = _sanitize_writeback_result(writeback, request) terminal_state = writeback.status + terminal_prompt_hashes = dict(writeback.after_hashes or prompt_snapshot.hashes()) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=terminal_prompt_hashes, + context="immediately after source writeback", + artifact_paths=artifact_paths, + journal=committing_journal, + request=request, + ) final_journal = _terminal_journal( committing_journal, writeback, @@ -543,8 +628,32 @@ async def _execute_with_snapshots( except Exception: pass raise + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=terminal_prompt_hashes, + context="after persisting terminal writeback journal", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ) persist_writeback_outcome(final_report, artifact_paths) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=terminal_prompt_hashes, + context="immediately before finalizing writeback run", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ) finalize_run_artifacts(final_report, artifact_paths) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=terminal_prompt_hashes, + context="immediately before returning writeback run", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ) return final_report @@ -767,6 +876,96 @@ def _current_prompt_hashes(snapshot: Any) -> dict[str, str]: return hashes +def _verify_prompt_integrity( + snapshot: Any, + *, + context: str, + expected_hashes: Mapping[str, str] | None = None, +) -> None: + expected = dict(expected_hashes or snapshot.hashes()) + observed: dict[str, str] = {} + for name, prompt_file in snapshot.files.items(): + try: + observed[name] = hashlib.sha256(prompt_file.path.read_bytes()).hexdigest() + except OSError as error: + raise PromptRestorationError( + f"source prompt field {name!r} is unreadable {context}" + ) from error + if observed != expected: + changed = sorted( + name + for name in set(expected) | set(observed) + if expected.get(name) != observed.get(name) + ) + raise ConcurrentPromptUpdateError( + f"source prompt integrity changed {context}: {', '.join(changed)}" + ) + + +async def _await_backend_with_prompt_integrity( + operation: Callable[[], Awaitable[Any]], + *, + prompt_snapshot: Any, + context: str, +) -> Any: + """Run one backend await while making source-prompt drift run-fatal.""" + + _verify_prompt_integrity( + prompt_snapshot, + context=f"before {context}", + ) + try: + result = await operation() + except BaseException as backend_error: + try: + _verify_prompt_integrity( + prompt_snapshot, + context=f"after failed {context}", + ) + except (ConcurrentPromptUpdateError, PromptRestorationError) as integrity_error: + raise integrity_error from backend_error + raise + _verify_prompt_integrity( + prompt_snapshot, + context=f"after {context}", + ) + return result + + +def _verify_terminal_writeback_integrity( + snapshot: Any, + *, + expected_hashes: Mapping[str, str], + context: str, + artifact_paths: Any, + journal: dict[str, Any], + request: PipelineRequest, +) -> None: + try: + _verify_prompt_integrity( + snapshot, + context=context, + expected_hashes=expected_hashes, + ) + except (ConcurrentPromptUpdateError, PromptRestorationError) as integrity_error: + unknown_journal = dict(journal) + unknown_journal.update( + { + "state": "unknown", + "report_phase": "final", + "after_hashes": _current_prompt_hashes(snapshot), + "error": _sanitize_error_message(str(integrity_error), request), + } + ) + try: + persist_writeback_journal(artifact_paths, unknown_journal) + except Exception as persist_error: + add_note = getattr(integrity_error, "add_note", None) + if add_note is not None: + add_note(f"failed to persist unknown writeback state: {persist_error}") + raise + + def _terminal_journal( journal: dict[str, Any], writeback: WritebackResult, @@ -866,19 +1065,13 @@ def _recoverable_candidate_evaluation_failure( (ConcurrentPromptUpdateError, PromptRestorationError), ): raise error - - expected_prompt_hashes = prompt_snapshot.hashes() - observed_prompt_hashes = _current_prompt_hashes(prompt_snapshot) - if observed_prompt_hashes != expected_prompt_hashes: - changed = sorted( - name - for name in set(expected_prompt_hashes) | set(observed_prompt_hashes) - if expected_prompt_hashes.get(name) != observed_prompt_hashes.get(name) + try: + _verify_prompt_integrity( + prompt_snapshot, + context="after failed candidate evaluation", ) - raise ConcurrentPromptUpdateError( - "source prompt integrity changed during failed candidate evaluation: " - + ", ".join(changed) - ) from error + except (ConcurrentPromptUpdateError, PromptRestorationError) as integrity_error: + raise integrity_error from error raw_message = str(error) return { @@ -1547,6 +1740,21 @@ def _powershell_quote(value: str) -> str: def _powershell_arg(value: str) -> str: + placeholder = re.search(r"\$(OUTPUT_DIR|EXTERNAL)", value) + if placeholder is not None: + prefix = value[: placeholder.start()] + suffix = value[placeholder.end() :] + + def escape_literal(text: str) -> str: + return text.replace("`", "``").replace("$", "`$").replace('"', '`"') + + return ( + '"' + + escape_literal(prefix) + + placeholder.group(0) + + escape_literal(suffix) + + '"' + ) if re.fullmatch(r"[A-Za-z0-9_./:-]+", value): return value return _powershell_quote(value) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index c645c892..1cb1dc89 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -6,6 +6,7 @@ import json import hashlib import os +import stat import sys import tempfile from dataclasses import dataclass @@ -278,14 +279,16 @@ def _durable_replace(source: Path, target: Path) -> None: """Atomically replace a file and wait for rename metadata to reach storage.""" if os.name == "nt": + if _is_reparse_point(target): + raise OSError(f"refusing to replace reparse-point target: {target}") move_file_ex = ctypes.WinDLL("kernel32", use_last_error=True).MoveFileExW move_file_ex.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32] move_file_ex.restype = ctypes.c_int movefile_replace_existing = 0x00000001 movefile_write_through = 0x00000008 succeeded = move_file_ex( - str(source.resolve()), - str(target.resolve()), + os.path.abspath(source), + os.path.abspath(target), movefile_replace_existing | movefile_write_through, ) if not succeeded: @@ -295,6 +298,15 @@ def _durable_replace(source: Path, target: Path) -> None: _fsync_directory(target.parent) +def _is_reparse_point(path: Path) -> bool: + try: + metadata = os.lstat(path) + except FileNotFoundError: + return False + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x00000400) + return bool(getattr(metadata, "st_file_attributes", 0) & reparse_flag) + + def render_markdown(report: OptimizationReport) -> str: decision_by_id = {decision.candidate_id: decision for decision in report.gate_decisions} lines = [ diff --git a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py index 4df6202d..d28dcb2a 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py @@ -15,6 +15,7 @@ from dataclasses import dataclass from pathlib import Path +from .artifacts import validate_distinct_file_paths from .schemas import WritebackResult @@ -55,6 +56,7 @@ def _hash_bytes(content: bytes) -> str: def snapshot_prompt_files(paths: dict[str, str | Path]) -> PromptSnapshot: """Read source prompt files and capture their exact bytes and hashes.""" + validate_distinct_file_paths(paths, context="prompt snapshot files") files: dict[str, PromptFileSnapshot] = {} for name, raw_path in paths.items(): path = Path(raw_path) diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 991d9aae..be0dbe33 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -21,6 +21,7 @@ try: # Package import during tests. from .eval_loop.artifacts import validate_artifact_component + from .eval_loop.artifacts import validate_distinct_file_paths from .eval_loop.backends import FakeBackend from .eval_loop.backends import SDKBackend from .eval_loop.config import _parse_gate_config @@ -32,6 +33,7 @@ from .eval_loop.schemas import OptimizationReport except ImportError: # Direct script execution. from eval_loop.artifacts import validate_artifact_component + from eval_loop.artifacts import validate_distinct_file_paths from eval_loop.backends import FakeBackend from eval_loop.backends import SDKBackend from eval_loop.config import _parse_gate_config @@ -170,10 +172,10 @@ def build_pipeline_request_and_backend( "--fake-judge or use --mode sdk with --sdk-call-agent module:function." ) - train_resolved = Path(train_path).resolve() - validation_resolved = Path(val_path).resolve() - if train_resolved == validation_resolved: - raise ValueError("train and validation evalset paths must be different") + validate_distinct_file_paths( + {"train": train_path, "validation": val_path}, + context="train and validation evalset paths", + ) target_prompt_paths = _parse_target_prompt_paths( target_prompts, @@ -282,7 +284,6 @@ def _parse_target_prompt_paths( return {"system_prompt": Path(default_prompt_path).resolve()} parsed: dict[str, str | Path] = {} casefold_names: set[str] = set() - resolved_paths: set[Path] = set() for item in target_prompts: if "=" not in item: raise ValueError("--target-prompt must use name=path format") @@ -303,12 +304,12 @@ def _parse_target_prompt_paths( raise ValueError(f"--target-prompt duplicate field name {name!r}") if name.casefold() in casefold_names: raise ValueError("--target-prompt field names must be case-insensitively unique") - resolved_path = Path(path).resolve() - if resolved_path in resolved_paths: - raise ValueError("--target-prompt fields must not reference the same resolved file") - resolved_paths.add(resolved_path) casefold_names.add(name.casefold()) - parsed[name] = resolved_path + parsed[name] = Path(path).resolve() + validate_distinct_file_paths( + parsed, + context="--target-prompt fields", + ) return parsed diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py index d958f4f4..7abd87d3 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import hashlib import json import math @@ -96,7 +97,7 @@ def __init__( self, *, fail_on: tuple[str, str], - failure: Exception, + failure: BaseException, delegate: RecordingBackend, ) -> None: super().__init__( @@ -140,6 +141,47 @@ async def evaluate(self, **kwargs: Any) -> EvalResult: return await super().evaluate(**kwargs) +class SourceMutatingBackend(RecordingBackend): + def __init__( + self, + *, + request: PipelineRequest, + mutate_on: tuple[str, str] | str, + failure: BaseException | None, + delegate: RecordingBackend, + ) -> None: + super().__init__( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + self.request = request + self.mutate_on = mutate_on + self.failure = failure + + def _mutate_source(self) -> None: + Path(self.request.target_prompt_paths["system_prompt"]).write_text( + "backend source mutation", + encoding="utf-8", + ) + + async def evaluate(self, **kwargs: Any) -> EvalResult: + result = await super().evaluate(**kwargs) + if (kwargs["prompt_id"], kwargs["split"]) == self.mutate_on: + self._mutate_source() + if self.failure is not None: + raise self.failure + return result + + async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: + result = await super().optimize_candidates(**kwargs) + if self.mutate_on == "optimizer": + self._mutate_source() + if self.failure is not None: + raise self.failure + return result + + @pytest.mark.asyncio async def test_execute_pipeline_uses_train_only_evidence_and_full_candidate_reevaluation(tmp_path: Path): request = _request(tmp_path, update_source=False) @@ -216,6 +258,114 @@ async def test_backend_cannot_mutate_canonical_baseline_or_candidate_bundles(tmp assert Path(request.target_prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "safe prompt" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("mutate_on", "update_source"), + [ + (("baseline", "train"), False), + (("baseline", "validation"), False), + ("optimizer", False), + (("candidate_a", "train"), False), + (("candidate_a", "validation"), True), + ], +) +async def test_successful_backend_source_mutation_is_fatal( + tmp_path: Path, + mutate_on: tuple[str, str] | str, + update_source: bool, +): + request = _request(tmp_path, update_source=update_source) + backend = SourceMutatingBackend( + request=request, + mutate_on=mutate_on, + failure=None, + delegate=_safe_backend(candidate_count=1), + ) + + with pytest.raises(ConcurrentPromptUpdateError, match="source prompt.*changed"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert Path(request.target_prompt_paths["system_prompt"]).read_text( + encoding="utf-8" + ) == "backend source mutation" + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure", + [RuntimeError("backend error after mutation"), asyncio.CancelledError()], + ids=["error", "cancellation"], +) +async def test_backend_error_or_cancellation_with_source_mutation_is_integrity_fatal( + tmp_path: Path, + failure: BaseException, +): + request = _request(tmp_path, update_source=False) + backend = SourceMutatingBackend( + request=request, + mutate_on=("candidate_a", "train"), + failure=failure, + delegate=_safe_backend(candidate_count=2), + ) + + with pytest.raises( + ConcurrentPromptUpdateError, + match="source prompt.*changed", + ) as error_info: + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert error_info.value.__cause__ is failure + assert not any( + call[0] == "evaluate" and call[1] == "candidate_b" + for call in backend.calls + ) + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +async def test_clean_backend_cancellation_preserves_cancelled_error(tmp_path: Path): + request = _request(tmp_path, update_source=False) + cancellation = asyncio.CancelledError() + backend = FailingCandidateBackend( + fail_on=("candidate_a", "train"), + failure=cancellation, + delegate=_safe_backend(candidate_count=2), + ) + + with pytest.raises(asyncio.CancelledError) as error_info: + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert error_info.value is cancellation + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + assert not any( + call[0] == "evaluate" and call[1] == "candidate_b" + for call in backend.calls + ) + + +@pytest.mark.asyncio +async def test_unreadable_source_prompt_is_integrity_fatal_without_path_leak(tmp_path: Path): + request = _request(tmp_path, update_source=False) + backend = SourceMutatingBackend( + request=request, + mutate_on=("baseline", "train"), + failure=None, + delegate=_safe_backend(candidate_count=1), + ) + + def remove_source() -> None: + Path(request.target_prompt_paths["system_prompt"]).unlink() + + backend._mutate_source = remove_source # type: ignore[method-assign] + + with pytest.raises(PromptRestorationError, match="source prompt field") as error_info: + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert str(tmp_path) not in str(error_info.value) + assert isinstance(error_info.value.__cause__, OSError) + + @pytest.mark.asyncio @pytest.mark.parametrize("failed_split", ["train", "validation"]) async def test_candidate_evaluation_error_rejects_candidate_and_continues( @@ -859,16 +1009,13 @@ async def test_writeback_journal_is_committing_before_source_commit( ): request = _request(tmp_path, update_source=True) backend = _safe_backend(candidate_count=1) + real_commit = pipeline_module.commit_prompt_bundle def commit(snapshot, prompts): journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" journal = json.loads(journal_path.read_text(encoding="utf-8")) assert journal["state"] == "committing" - return WritebackResult( - status="applied", - before_hashes=snapshot.hashes(), - after_hashes={name: hashlib.sha256(prompts[name].encode("utf-8")).hexdigest() for name in prompts}, - ) + return real_commit(snapshot, prompts) monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", commit) @@ -996,6 +1143,48 @@ async def test_backend_neutral_core_rejects_same_resolved_train_and_validation_p assert backend.calls == [] +@pytest.mark.asyncio +async def test_backend_neutral_core_rejects_hardlinked_train_and_validation( + tmp_path: Path, +): + request = _request(tmp_path) + validation_path = Path(request.validation_path) + validation_path.unlink() + os.link(request.train_path, validation_path) + assert Path(request.train_path).samefile(validation_path) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="train and validation.*different physical files"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + assert not (Path(request.output_dir) / "runs").exists() + + +def test_factory_rejects_hardlinked_train_and_validation_before_backend_creation( + tmp_path: Path, +): + request = _request(tmp_path) + validation_path = Path(request.validation_path) + validation_path.unlink() + os.link(request.train_path, validation_path) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="train and validation.*different physical files"): + build_pipeline_request_and_backend( + train_path=request.train_path, + val_path=validation_path, + optimizer_config_path=request.optimizer_config_path, + prompt_path=request.target_prompt_paths["system_prompt"], + output_dir=request.output_dir, + mode="fake", + backend=backend, + ) + + assert backend.calls == [] + assert not Path(request.output_dir).exists() + + @pytest.mark.asyncio @pytest.mark.parametrize("field_names", [["Foo", "foo"], ["CON"]]) async def test_backend_neutral_core_rejects_unsafe_or_colliding_target_fields( @@ -1029,12 +1218,55 @@ async def test_backend_neutral_core_rejects_target_fields_for_same_resolved_file ) backend = _safe_backend(candidate_count=1) - with pytest.raises(ValueError, match="target prompt fields.*same resolved file"): + with pytest.raises(ValueError, match="target prompt fields.*different physical files"): await execute_pipeline(request, evaluator=backend, optimizer=backend) assert backend.calls == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("alias_kind", ["hardlink", "casefold"]) +async def test_backend_neutral_core_rejects_physical_or_casefold_target_aliases( + tmp_path: Path, + alias_kind: str, +): + request = _request(tmp_path) + if alias_kind == "hardlink": + first_path = Path(request.target_prompt_paths["system_prompt"]) + second_path = tmp_path / "prompt-hardlink.txt" + os.link(first_path, second_path) + assert first_path.samefile(second_path) + else: + first_path = tmp_path / "CasePrompt.txt" + second_path = tmp_path / "caseprompt.txt" + first_path.write_text("first", encoding="utf-8") + second_path.write_text("second", encoding="utf-8") + request = replace( + request, + target_prompt_paths={"system_prompt": first_path, "router_prompt": second_path}, + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="target prompt fields.*different physical files"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + assert not (Path(request.output_dir) / "runs").exists() + + +def test_shared_path_validator_handles_missing_casefold_collisions(tmp_path: Path): + pipeline_module.validate_distinct_file_paths( + {"first": tmp_path / "first.txt", "second": tmp_path / "second.txt"}, + context="test paths", + ) + + with pytest.raises(ValueError, match="case-insensitively"): + pipeline_module.validate_distinct_file_paths( + {"first": tmp_path / "Missing.txt", "second": tmp_path / "missing.txt"}, + context="test paths", + ) + + @pytest.mark.asyncio async def test_backend_neutral_core_rejects_fake_backend_seed_mismatch(tmp_path: Path): request = _request(tmp_path) @@ -1443,6 +1675,145 @@ def tracked_durable_replace(source: Path, target: Path) -> None: assert not list(tmp_path.glob("*.tmp")) +def test_powershell_reproducibility_placeholders_remain_expandable(tmp_path: Path): + request = _request(tmp_path) + router_path = tmp_path / "router prompt.txt" + router_path.write_text("router", encoding="utf-8") + request = replace( + request, + target_prompt_paths={ + "system_prompt": request.target_prompt_paths["system_prompt"], + "router_prompt": router_path, + }, + ) + + command = pipeline_module._reproducibility_command(request) + + assert '--output-dir "$OUTPUT_DIR"' in command + assert '"$EXTERNAL/train.evalset.json"' in command + assert '"router_prompt=$EXTERNAL/router prompt.txt"' in command + assert "'$OUTPUT_DIR'" not in command + assert "'$EXTERNAL" not in command + + +@pytest.mark.skipif(os.name != "nt", reason="Windows MoveFileExW contract") +def test_windows_durable_replace_does_not_resolve_target( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + source = tmp_path / "source.tmp" + target = tmp_path / "target.json" + source.write_text("content", encoding="utf-8") + calls: list[tuple[str, str, int]] = [] + + class FakeMoveFileEx: + argtypes: object = None + restype: object = None + + def __call__(self, source_name: str, target_name: str, flags: int) -> int: + calls.append((source_name, target_name, flags)) + return 1 + + fake_move = FakeMoveFileEx() + fake_kernel = type("FakeKernel", (), {"MoveFileExW": fake_move})() + monkeypatch.setattr(report_module.ctypes, "WinDLL", lambda *args, **kwargs: fake_kernel) + monkeypatch.setattr(report_module, "_is_reparse_point", lambda path: False) + real_resolve = Path.resolve + + def reject_target_resolve(path: Path, *args: Any, **kwargs: Any) -> Path: + if path == target: + raise AssertionError("durable replace must not resolve the target leaf") + return real_resolve(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", reject_target_resolve) + + report_module._durable_replace(source, target) + + assert calls + assert calls[0][1] == str(target.absolute()) + + +@pytest.mark.skipif(os.name != "nt", reason="Windows reparse-point contract") +def test_windows_durable_replace_rejects_existing_reparse_target( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + source = tmp_path / "source.tmp" + target = tmp_path / "target.json" + source.write_text("content", encoding="utf-8") + target.write_text("old", encoding="utf-8") + monkeypatch.setattr(report_module, "_is_reparse_point", lambda path: path == target) + monkeypatch.setattr( + report_module.ctypes, + "WinDLL", + lambda *args, **kwargs: pytest.fail("MoveFileExW must not follow a reparse target"), + ) + + with pytest.raises(OSError, match="reparse-point target"): + report_module._durable_replace(source, target) + + +@pytest.mark.asyncio +async def test_prompt_integrity_is_checked_immediately_before_prepare( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=False) + backend = _safe_backend(candidate_count=1) + original_build_report = pipeline_module.build_report + + def mutate_after_report_build(**kwargs: Any): + report = original_build_report(**kwargs) + Path(request.target_prompt_paths["system_prompt"]).write_text( + "late source mutation", + encoding="utf-8", + ) + return report + + monkeypatch.setattr(pipeline_module, "build_report", mutate_after_report_build) + + with pytest.raises(ConcurrentPromptUpdateError, match="source prompt integrity changed"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + run_dir = Path(request.output_dir) / "runs" / request.run_id + assert not (run_dir / "pre_write_report.json").exists() + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +async def test_late_no_write_source_mutation_downgrades_journal_to_unknown( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=False) + backend = _safe_backend(candidate_count=1) + original_finalize = pipeline_module.finalize_run_artifacts + + def mutate_after_finalize(report, paths): + original_finalize(report, paths) + Path(request.target_prompt_paths["system_prompt"]).write_text( + "mutation after final artifacts", + encoding="utf-8", + ) + + monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", mutate_after_finalize) + + with pytest.raises(ConcurrentPromptUpdateError, match="source prompt integrity changed"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + journal_path = ( + Path(request.output_dir) + / "runs" + / request.run_id + / "writeback_journal.json" + ) + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert journal["state"] == "unknown" + assert journal["after_hashes"] == { + "system_prompt": hashlib.sha256(b"mutation after final artifacts").hexdigest() + } + + @pytest.mark.asyncio async def test_prepare_writes_completion_journal_after_all_other_artifacts( tmp_path: Path, diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index d60c32e9..565ce196 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -3,6 +3,7 @@ import builtins import inspect import json +import os import shlex import sys import types @@ -1616,7 +1617,24 @@ def test_target_prompt_paths_reject_same_resolved_file(tmp_path: Path): default_prompt_path=prompt_path, ) - assert str(exc_info.value) == "--target-prompt fields must not reference the same resolved file" + assert "--target-prompt fields must be different physical files" in str(exc_info.value) + + +def test_target_prompt_paths_reject_hardlink_aliases(tmp_path: Path): + prompt_path = tmp_path / "prompt.txt" + hardlink_path = tmp_path / "prompt-hardlink.txt" + prompt_path.write_text("baseline", encoding="utf-8") + os.link(prompt_path, hardlink_path) + assert prompt_path.samefile(hardlink_path) + + with pytest.raises(ValueError, match="different physical files"): + _parse_target_prompt_paths( + [ + f"system_prompt={prompt_path}", + f"router_prompt={hardlink_path}", + ], + default_prompt_path=prompt_path, + ) @pytest.mark.parametrize( diff --git a/examples/optimization/eval_optimize_loop/tests/test_writeback.py b/examples/optimization/eval_optimize_loop/tests/test_writeback.py index 7d2fbb30..dad140c0 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_writeback.py +++ b/examples/optimization/eval_optimize_loop/tests/test_writeback.py @@ -52,6 +52,17 @@ def test_snapshot_prompt_files_preserves_bytes_and_returns_hash_copy(tmp_path: P assert snapshot.hashes()["system"] == system.sha256 +def test_snapshot_prompt_files_rejects_hardlink_aliases(tmp_path: Path): + prompt_path = tmp_path / "prompt.txt" + alias_path = tmp_path / "prompt-hardlink.txt" + prompt_path.write_text("baseline", encoding="utf-8") + os.link(prompt_path, alias_path) + assert prompt_path.samefile(alias_path) + + with pytest.raises(ValueError, match="different physical files"): + snapshot_prompt_files({"system": prompt_path, "router": alias_path}) + + def test_temporary_prompt_bundle_restores_exact_bytes_after_body_error(tmp_path: Path): paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) From 97394aa9d2d5782c77a143e1230e803bda4dffa7 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 04:25:23 +0800 Subject: [PATCH 28/34] fix(examples): make optimization audit reproducible --- .../eval_optimize_loop/eval_loop/pipeline.py | 104 ++- .../eval_optimize_loop/eval_loop/report.py | 679 ++++++++++++++-- .../eval_optimize_loop/eval_loop/schemas.py | 1 + .../tests/test_config_validation.py | 25 + .../tests/test_pipeline_orchestration.py | 62 +- .../tests/test_report_artifacts.py | 722 ++++++++++++++++++ .../tests/test_sdk_backend.py | 6 +- 7 files changed, 1445 insertions(+), 154 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py diff --git a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py index a3f9a1b2..f605cbed 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py @@ -29,12 +29,15 @@ from .gate import AcceptanceGate from .loader import read_json from .loader import stable_config_hash +from .report import RunArtifactPaths from .report import build_report from .report import compute_case_deltas from .report import finalize_run_artifacts from .report import persist_writeback_journal from .report import persist_writeback_outcome from .report import prepare_run_artifacts +from .report import reserve_run_artifacts +from .report import validate_unique_round_ids from .schemas import CandidatePrompt from .schemas import CostSummary from .schemas import EvalResult @@ -119,7 +122,8 @@ async def execute_pipeline( {"train": request.train_path, "validation": request.validation_path}, context="train and validation evalset paths", ) - run_root = _claim_run_directory(request.output_dir, run_id=run_id) + artifact_paths = reserve_run_artifacts(request.output_dir, run_id=run_id) + run_root = artifact_paths.temp_run_dir input_snapshots = _snapshot_pipeline_inputs(request, run_id=run_id) try: @@ -130,6 +134,7 @@ async def execute_pipeline( started=started, run_id=run_id, run_root=run_root, + artifact_paths=artifact_paths, input_snapshots=input_snapshots, ) except BaseException as primary_error: @@ -172,6 +177,7 @@ async def _execute_with_snapshots( started: float, run_id: str, run_root: Path, + artifact_paths: RunArtifactPaths, input_snapshots: _InputSnapshots, ) -> OptimizationReport: prompt_snapshot = snapshot_prompt_files(request.target_prompt_paths) @@ -483,6 +489,7 @@ async def _execute_with_snapshots( gate_decisions=gate_decisions, selected_candidate=selected_candidate, audit=audit, + baseline_prompts=dict(baseline_prompts), rounds=optimization.rounds, cost_summary=cost_summary, writeback=provisional_writeback, @@ -492,7 +499,7 @@ async def _execute_with_snapshots( prompt_snapshot, context="immediately before preparing run artifacts", ) - artifact_paths = prepare_run_artifacts(report, request.output_dir) + prepare_run_artifacts(report, artifact_paths) if selected_candidate is None or not request.update_source: terminal_journal = _terminal_journal( writeback_journal, @@ -509,22 +516,17 @@ async def _execute_with_snapshots( request=request, ) persist_writeback_outcome(final_report, artifact_paths) - _verify_terminal_writeback_integrity( - prompt_snapshot, - expected_hashes=prompt_snapshot.hashes(), - context="immediately before finalizing no-write run", - artifact_paths=artifact_paths, - journal=terminal_journal, - request=request, - ) - finalize_run_artifacts(final_report, artifact_paths) - _verify_terminal_writeback_integrity( - prompt_snapshot, - expected_hashes=prompt_snapshot.hashes(), - context="immediately before returning no-write run", - artifact_paths=artifact_paths, - journal=terminal_journal, - request=request, + finalize_run_artifacts( + final_report, + artifact_paths, + before_publish=lambda: _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="immediately before publishing no-write run", + artifact_paths=artifact_paths, + journal=terminal_journal, + request=request, + ), ) return final_report @@ -553,22 +555,17 @@ async def _execute_with_snapshots( request=request, ) persist_writeback_outcome(final_report, artifact_paths) - _verify_terminal_writeback_integrity( - prompt_snapshot, - expected_hashes=prompt_snapshot.hashes(), - context="immediately before finalizing input-drift run", - artifact_paths=artifact_paths, - journal=final_journal, - request=request, - ) - finalize_run_artifacts(final_report, artifact_paths) - _verify_terminal_writeback_integrity( - prompt_snapshot, - expected_hashes=prompt_snapshot.hashes(), - context="immediately before returning input-drift run", - artifact_paths=artifact_paths, - journal=final_journal, - request=request, + finalize_run_artifacts( + final_report, + artifact_paths, + before_publish=lambda: _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="immediately before publishing input-drift run", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ), ) return final_report @@ -637,22 +634,17 @@ async def _execute_with_snapshots( request=request, ) persist_writeback_outcome(final_report, artifact_paths) - _verify_terminal_writeback_integrity( - prompt_snapshot, - expected_hashes=terminal_prompt_hashes, - context="immediately before finalizing writeback run", - artifact_paths=artifact_paths, - journal=final_journal, - request=request, - ) - finalize_run_artifacts(final_report, artifact_paths) - _verify_terminal_writeback_integrity( - prompt_snapshot, - expected_hashes=terminal_prompt_hashes, - context="immediately before returning writeback run", - artifact_paths=artifact_paths, - journal=final_journal, - request=request, + finalize_run_artifacts( + final_report, + artifact_paths, + before_publish=lambda: _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=terminal_prompt_hashes, + context="immediately before publishing writeback run", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ), ) return final_report @@ -711,17 +703,6 @@ def _snapshot_pipeline_inputs(request: PipelineRequest, *, run_id: str) -> _Inpu return _InputSnapshots(snapshots) -def _claim_run_directory(output_dir: str | Path, *, run_id: str) -> Path: - runs_root = Path(output_dir) / "runs" - runs_root.mkdir(parents=True, exist_ok=True) - run_root = runs_root / run_id - try: - run_root.mkdir() - except FileExistsError as error: - raise ValueError(f"run_id {run_id!r} already exists in {runs_root}") from error - return run_root - - def _verify_snapshot_integrity( snapshots: _InputSnapshots, *requested_roles: str, @@ -1335,6 +1316,7 @@ def _validate_optimization_result(result: OptimizationResult) -> None: round_record.cost, context=f"optimization round {round_record.round_id} cost", ) + validate_unique_round_ids(result.rounds) if not isinstance(result.raw_summary, dict): raise ValueError("optimization raw_summary must be an object") _validate_nested_numbers(result.raw_summary, context="optimization raw_summary") diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index 1cb1dc89..a3c18d4f 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -3,12 +3,14 @@ from __future__ import annotations import ctypes +import errno import json import hashlib import os import stat import sys import tempfile +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from typing import Any @@ -42,16 +44,54 @@ class RunArtifactPaths: """Locations written by the audit-first report lifecycle.""" output_dir: Path - run_dir: Path - prewrite_json: Path - prewrite_markdown: Path - audit_json: Path - final_json: Path - final_markdown: Path - writeback_json: Path - writeback_journal: Path - top_level_json: Path - top_level_markdown: Path + run_id: str + temp_run_dir: Path + final_run_dir: Path + + @property + def run_dir(self) -> Path: + """Return the immutable authoritative run location.""" + + return self.final_run_dir + + @property + def prewrite_json(self) -> Path: + return self.final_run_dir / "pre_write_report.json" + + @property + def prewrite_markdown(self) -> Path: + return self.final_run_dir / "pre_write_report.md" + + @property + def audit_json(self) -> Path: + return self.final_run_dir / "audit.json" + + @property + def final_json(self) -> Path: + return self.final_run_dir / "optimization_report.json" + + @property + def final_markdown(self) -> Path: + return self.final_run_dir / "optimization_report.md" + + @property + def writeback_json(self) -> Path: + return self.final_run_dir / "writeback.json" + + @property + def writeback_journal(self) -> Path: + return self.final_run_dir / "writeback_journal.json" + + @property + def top_level_json(self) -> Path: + return self.output_dir / "optimization_report.json" + + @property + def top_level_markdown(self) -> Path: + return self.output_dir / "optimization_report.md" + + def temp_path(self, relative_path: str | Path) -> Path: + return self.temp_run_dir / relative_path def compute_case_deltas( @@ -100,6 +140,7 @@ def build_report( gate_decisions: list[GateDecision], selected_candidate: str | None, audit: dict[str, Any], + baseline_prompts: dict[str, str] | None = None, rounds: list[OptimizationRound] | None = None, cost_summary: CostSummary | None = None, writeback: WritebackResult | None = None, @@ -123,90 +164,209 @@ def build_report( gate_decisions=gate_decisions, selected_candidate=selected_candidate, audit=audit, + baseline_prompts=dict(baseline_prompts or {}), rounds=list(rounds or []), cost_summary=cost_summary or CostSummary(), writeback=writeback or WritebackResult(status="not_requested"), ) +def validate_unique_round_ids(rounds: list[OptimizationRound]) -> None: + """Reject round collections that cannot map one-to-one to audit files.""" + + seen: set[int] = set() + for round_record in rounds: + round_id = round_record.round_id + if round_id in seen: + raise ValueError(f"duplicate round_id in optimization report: {round_id}") + seen.add(round_id) + + def write_reports(report: OptimizationReport, output_dir: str | Path) -> tuple[Path, Path]: """Compatibility helper for callers that do not perform source writeback.""" - paths = prepare_run_artifacts(report, output_dir) + run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) + paths = reserve_run_artifacts(output_dir, run_id=run_id) + prepare_run_artifacts(report, paths) finalize_run_artifacts(report, paths) return paths.top_level_json, paths.top_level_markdown -def prepare_run_artifacts( - report: OptimizationReport, +def reserve_run_artifacts( output_dir: str | Path, + *, + run_id: str, ) -> RunArtifactPaths: - """Persist a complete pre-write audit before any source prompt commit.""" + """Exclusively reserve one mutable temp run without touching a final run.""" + safe_run_id = _safe_artifact_name(str(run_id)) output_path = Path(output_dir) - run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) - run_dir = output_path / "runs" / run_id - run_dir.mkdir(parents=True, exist_ok=True) - paths = RunArtifactPaths( + runs_root = output_path / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + if runs_root.is_symlink() or _is_reparse_point(runs_root): + raise OSError(f"refusing to reserve run artifacts through unsafe runs directory: {runs_root}") + + temp_run_dir = runs_root / f".{safe_run_id}.tmp" + final_run_dir = runs_root / safe_run_id + for occupied in (final_run_dir, temp_run_dir): + if os.path.lexists(occupied): + raise FileExistsError( + f"run ID {safe_run_id!r} is already reserved or published in {runs_root}" + ) + try: + temp_run_dir.mkdir() + except FileExistsError as error: + raise FileExistsError( + f"run ID {safe_run_id!r} is already reserved or published in {runs_root}" + ) from error + + # A concurrently published final must win. Leave no misleading reservation + # when our just-created directory is still empty. + if os.path.lexists(final_run_dir): + try: + temp_run_dir.rmdir() + except OSError: + pass + raise FileExistsError( + f"run ID {safe_run_id!r} is already reserved or published in {runs_root}" + ) + return RunArtifactPaths( output_dir=output_path, - run_dir=run_dir, - prewrite_json=run_dir / "pre_write_report.json", - prewrite_markdown=run_dir / "pre_write_report.md", - audit_json=run_dir / "audit.json", - final_json=run_dir / "optimization_report.json", - final_markdown=run_dir / "optimization_report.md", - writeback_json=run_dir / "writeback.json", - writeback_journal=run_dir / "writeback_journal.json", - top_level_json=output_path / "optimization_report.json", - top_level_markdown=output_path / "optimization_report.md", + run_id=safe_run_id, + temp_run_dir=temp_run_dir, + final_run_dir=final_run_dir, ) - _atomic_write_text(paths.prewrite_json, report_to_json(report)) - _atomic_write_text(paths.prewrite_markdown, render_markdown(report)) - _atomic_write_text(paths.audit_json, _json_text(report.audit)) - write_audit_artifacts(report, output_path) + + +def prepare_run_artifacts( + report: OptimizationReport, + paths_or_output_dir: RunArtifactPaths | str | Path, +) -> RunArtifactPaths: + """Persist a complete pre-write audit before any source prompt commit.""" + + report_run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) + if isinstance(paths_or_output_dir, RunArtifactPaths): + paths = paths_or_output_dir + else: + # Legacy direct callers remain safe: they receive a new exclusive + # reservation and can never reopen an existing temp or final run. + paths = reserve_run_artifacts(paths_or_output_dir, run_id=report_run_id) + validate_unique_round_ids(report.rounds) + _validate_run_artifact_paths(paths, report_run_id=report_run_id) + _require_mutable_temp_run(paths) + + prewrite_json = report_to_json(report) + prewrite_markdown = render_markdown(report) + audit_json = _json_text(report.audit) journal = dict(report.audit.get("writeback_journal", {})) if journal.get("state") == "pending": journal["state"] = "prepared" + journal_json = _json_text(journal) + + _atomic_write_text(paths.temp_path("pre_write_report.json"), prewrite_json) + _atomic_write_text(paths.temp_path("pre_write_report.md"), prewrite_markdown) + _atomic_write_text(paths.temp_path("audit.json"), audit_json) + write_audit_artifacts(report, paths.temp_run_dir) + _write_artifact_manifest(report, paths, expected_journal=journal_json) # This durable journal is the completion marker for the whole prepare phase, # so it must be replaced only after every prerequisite artifact exists. persist_writeback_journal(paths, journal) return paths -def finalize_run_artifacts(report: OptimizationReport, paths: RunArtifactPaths) -> None: - """Persist final writeback state and refresh top-level convenience copies.""" - - expected_run_dir = paths.output_dir / "runs" / _safe_artifact_name( - str(report.run.get("run_id") or "run") - ) - if paths.run_dir != expected_run_dir: - raise ValueError("run artifact paths do not match report run_id") - # The terminal outcome is authoritative and is persisted before convenience - # reports. A later rendering failure cannot erase knowledge of source state. +def finalize_run_artifacts( + report: OptimizationReport, + paths: RunArtifactPaths, + *, + before_publish: Callable[[], None] | None = None, +) -> None: + """Complete the temp run, publish it once, then refresh convenience copies.""" + + report_run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) + validate_unique_round_ids(report.rounds) + _validate_run_artifact_paths(paths, report_run_id=report_run_id) + _require_mutable_temp_run(paths) + + # Serialize everything before the first final-phase write so non-finite or + # otherwise invalid values cannot create a misleading terminal artifact set. + audit_json = _json_text(report.audit) + final_json = report_to_json(report) + final_markdown = render_markdown(report) + + # The terminal outcome is authoritative within the still-mutable temp run. persist_writeback_outcome(report, paths) - _atomic_write_text(paths.audit_json, _json_text(report.audit)) - _atomic_write_text(paths.final_json, report_to_json(report)) - _atomic_write_text(paths.final_markdown, render_markdown(report)) - paths.output_dir.mkdir(parents=True, exist_ok=True) - _atomic_write_text(paths.top_level_json, report_to_json(report)) - _atomic_write_text(paths.top_level_markdown, render_markdown(report)) - write_audit_artifacts(report, paths.output_dir) + _atomic_write_text(paths.temp_path("audit.json"), audit_json) + _atomic_write_text(paths.temp_path("optimization_report.json"), final_json) + _atomic_write_text(paths.temp_path("optimization_report.md"), final_markdown) + write_audit_artifacts(report, paths.temp_run_dir) + _write_artifact_manifest(report, paths) + + # This is intentionally the last callback before the one-way directory + # rename. It may downgrade the temp journal and raise, but publication makes + # every final-run byte immutable to this lifecycle. + if before_publish is not None: + try: + before_publish() + except BaseException as callback_error: + try: + _write_artifact_manifest(report, paths) + except BaseException as manifest_error: + add_note = getattr(callback_error, "add_note", None) + if add_note is not None: + add_note( + "failed to refresh temp artifact manifest after before_publish failure: " + f"{manifest_error}" + ) + raise + _durable_publish_directory(paths) + + # Convenience copies are non-authoritative and are refreshed only from the + # already-published immutable run. Failure here leaves that final intact. + _atomic_write_bytes(paths.top_level_json, paths.final_json.read_bytes()) + _atomic_write_bytes(paths.top_level_markdown, paths.final_markdown.read_bytes()) def persist_writeback_journal(paths: RunArtifactPaths, journal: dict[str, Any]) -> None: """Atomically persist the authoritative writeback state machine record.""" - _atomic_write_text(paths.writeback_journal, _json_text(journal)) + _require_mutable_temp_run(paths) + _atomic_write_text(paths.temp_path("writeback_journal.json"), _json_text(journal)) def persist_writeback_outcome(report: OptimizationReport, paths: RunArtifactPaths) -> None: """Persist a terminal writeback outcome before any nonessential artifact.""" persist_writeback_journal(paths, dict(report.audit.get("writeback_journal", {}))) - _atomic_write_text(paths.writeback_json, _json_text(report.writeback)) + _atomic_write_text(paths.temp_path("writeback.json"), _json_text(report.writeback)) + + +def _validate_run_artifact_paths(paths: RunArtifactPaths, *, report_run_id: str) -> None: + runs_root = paths.output_dir / "runs" + if paths.run_id != report_run_id: + raise ValueError("run artifact paths do not match report run_id") + if paths.temp_run_dir != runs_root / f".{report_run_id}.tmp": + raise ValueError("run artifact temp path does not match report run_id") + if paths.final_run_dir != runs_root / report_run_id: + raise ValueError("run artifact final path does not match report run_id") + + +def _require_mutable_temp_run(paths: RunArtifactPaths) -> None: + if os.path.lexists(paths.final_run_dir): + raise FileExistsError(f"run ID {paths.run_id!r} is already published") + try: + metadata = os.lstat(paths.temp_run_dir) + except FileNotFoundError as error: + raise FileNotFoundError( + f"reserved temp directory for run ID {paths.run_id!r} is unavailable" + ) from error + if paths.temp_run_dir.is_symlink() or _is_reparse_point(paths.temp_run_dir): + raise OSError(f"refusing unsafe reserved temp directory: {paths.temp_run_dir}") + if not stat.S_ISDIR(metadata.st_mode): + raise OSError(f"reserved temp path is not a directory: {paths.temp_run_dir}") def report_to_json(report: OptimizationReport) -> str: + validate_unique_round_ids(report.rounds) return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True, allow_nan=False) + "\n" @@ -298,6 +458,85 @@ def _durable_replace(source: Path, target: Path) -> None: _fsync_directory(target.parent) +def _durable_publish_directory(paths: RunArtifactPaths) -> None: + """Atomically move the reserved temp run to a never-replaced final path.""" + + _require_mutable_temp_run(paths) + source = paths.temp_run_dir + target = paths.final_run_dir + if target.is_symlink() or _is_reparse_point(target): + raise OSError(f"refusing to publish over unsafe/reparse target: {target}") + if os.path.lexists(target): + raise FileExistsError(f"run ID {paths.run_id!r} is already published") + + if os.name == "nt": + move_file_ex = ctypes.WinDLL("kernel32", use_last_error=True).MoveFileExW + move_file_ex.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32] + move_file_ex.restype = ctypes.c_int + movefile_write_through = 0x00000008 + succeeded = move_file_ex( + os.path.abspath(source), + os.path.abspath(target), + movefile_write_through, + ) + if not succeeded: + error_code = ctypes.get_last_error() + if os.path.lexists(target): + raise FileExistsError( + error_code, + f"run ID {paths.run_id!r} is already published", + str(target), + ) + raise ctypes.WinError(error_code) + return + + _posix_rename_noreplace(source, target) + _fsync_directory(target.parent) + + +def _posix_rename_noreplace(source: Path, target: Path) -> None: + """Atomically rename a Linux directory without replacing any target.""" + + try: + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = libc.renameat2 + except (AttributeError, OSError) as error: + raise OSError( + errno.ENOTSUP, + "atomic POSIX no-replace directory rename is unavailable", + str(target), + ) from error + + at_fdcwd = -100 + rename_noreplace = 0x1 + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + at_fdcwd, + os.fsencode(source), + at_fdcwd, + os.fsencode(target), + rename_noreplace, + ) + if result == 0: + return + + error_number = ctypes.get_errno() or errno.EIO + if error_number in {errno.EEXIST, errno.ENOTEMPTY}: + raise FileExistsError( + error_number, + os.strerror(error_number), + str(target), + ) + raise OSError(error_number, os.strerror(error_number), str(target)) + + def _is_reparse_point(path: Path) -> bool: try: metadata = os.lstat(path) @@ -308,6 +547,7 @@ def _is_reparse_point(path: Path) -> bool: def render_markdown(report: OptimizationReport) -> str: + validate_unique_round_ids(report.rounds) decision_by_id = {decision.candidate_id: decision for decision in report.gate_decisions} lines = [ "# Evaluation + Optimization Report", @@ -471,48 +711,345 @@ def render_markdown(report: OptimizationReport) -> str: return "\n".join(lines) -def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None: - run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) - run_dir = output_path / "runs" / run_id - run_dir.mkdir(parents=True, exist_ok=True) +def write_audit_artifacts(report: OptimizationReport, run_dir: Path) -> None: + """Write the complete structured audit set inside a mutable run root.""" + + validate_unique_round_ids(report.rounds) + run_dir = Path(run_dir) + _require_safe_audit_root(run_dir) + + for component in ( + "case_results", + "baseline_prompts", + "candidate_prompts", + "prompt_diffs", + "rounds", + ): + _ensure_safe_audit_directory(run_dir, component) + for index, record in enumerate(report.candidates, start=1): + candidate: CandidatePrompt = record["candidate"] + _ensure_safe_audit_directory( + run_dir, + "candidate_prompts", + _candidate_artifact_name(index, candidate.candidate_id), + ) # Never copy the raw optimizer file: it may contain API keys. The original # byte hash remains in input_hashes while this human-readable snapshot is redacted. - _atomic_write_text( - run_dir / "config.snapshot.json", + _write_safe_audit_text( + run_dir, + "config.snapshot.json", _json_text(report.audit.get("config_snapshot", {})), ) - _atomic_write_text( - run_dir / "input_hashes.json", + _write_safe_audit_text( + run_dir, + "input_hashes.json", _json_text(report.audit.get("input_hashes", {})), ) - prompt_dir = run_dir / "candidate_prompts" - results_dir = run_dir / "case_results" - diffs_dir = run_dir / "prompt_diffs" - prompt_dir.mkdir(exist_ok=True) - results_dir.mkdir(exist_ok=True) - diffs_dir.mkdir(exist_ok=True) + for field_name, prompt_text in report.baseline_prompts.items(): + field_artifact = _safe_artifact_name(str(field_name)) + _write_safe_audit_text( + run_dir, + Path("baseline_prompts") / f"{field_artifact}.txt", + prompt_text, + ) + + _write_safe_audit_text( + run_dir, + Path("case_results") / "baseline_train.json", + _json_text(report.baseline_train), + ) + _write_safe_audit_text( + run_dir, + Path("case_results") / "baseline_validation.json", + _json_text(report.baseline_validation), + ) + _write_safe_audit_text(run_dir, "rounds.json", _json_text(report.rounds)) + for round_record in report.rounds: + round_artifact = _safe_artifact_name(str(round_record.round_id)) + _write_safe_audit_text( + run_dir, + Path("rounds") / f"{round_artifact}.json", + _json_text(round_record), + ) + _write_safe_audit_text( + run_dir, + "per_case_deltas.json", + _json_text(report.per_case_deltas), + ) + _write_safe_audit_text( + run_dir, + "gate_decisions.json", + _json_text(report.gate_decisions), + ) + _write_safe_audit_text( + run_dir, + "evaluation_failures.json", + _json_text(report.audit.get("candidate_evaluation_failures", {})), + ) + _write_safe_audit_text(run_dir, "writeback.json", _json_text(report.writeback)) for index, record in enumerate(report.candidates, start=1): candidate: CandidatePrompt = record["candidate"] candidate_name = _candidate_artifact_name(index, candidate.candidate_id) - candidate_dir = prompt_dir / candidate_name - candidate_dir.mkdir(exist_ok=True) prompt_bundle = record.get("prompt_bundle") if prompt_bundle is None: prompt_bundle = candidate.bundle() for field_name, prompt_text in prompt_bundle.items(): field_artifact = _safe_artifact_name(str(field_name)) - _atomic_write_text(candidate_dir / f"{field_artifact}.txt", prompt_text) - _atomic_write_text(diffs_dir / f"{candidate_name}.diff", candidate.prompt_diff) + _write_safe_audit_text( + run_dir, + Path("candidate_prompts") / candidate_name / f"{field_artifact}.txt", + prompt_text, + ) + _write_safe_audit_text( + run_dir, + Path("prompt_diffs") / f"{candidate_name}.diff", + candidate.prompt_diff, + ) for split_name in ("train_result", "validation_result"): split_result = record.get(split_name) if not isinstance(split_result, EvalResult): continue split_artifact = _safe_artifact_name(str(split_result.split)) - path = results_dir / f"{candidate_name}_{split_artifact}.json" - _atomic_write_text(path, _json_text(split_result)) + _write_safe_audit_text( + run_dir, + Path("case_results") / f"{candidate_name}_{split_artifact}.json", + _json_text(split_result), + ) + + +def _require_safe_audit_root(run_root: Path) -> None: + run_root = Path(run_root) + try: + metadata = os.lstat(run_root) + except FileNotFoundError as error: + raise OSError(f"audit run root is unavailable: {run_root}") from error + if stat.S_ISLNK(metadata.st_mode) or _is_reparse_point(run_root): + raise OSError(f"audit run root is unsafe/reparse: {run_root}") + if not stat.S_ISDIR(metadata.st_mode): + raise OSError(f"audit run root is not a directory: {run_root}") + + +def _write_safe_audit_text( + run_root: Path, + relative_path: str | Path, + content: str, +) -> None: + """Recheck an audit file's root and parent immediately before writing.""" + + run_root = Path(run_root) + relative = Path(relative_path) + if ( + relative.is_absolute() + or not relative.parts + or any(component in {"", ".", ".."} for component in relative.parts) + ): + raise OSError(f"unsafe audit artifact path: {relative_path!r}") + if len(relative.parts) == 1: + _require_safe_audit_root(run_root) + parent = run_root + else: + parent = _ensure_safe_audit_directory(run_root, *relative.parts[:-1]) + target = parent / relative.parts[-1] + try: + target.relative_to(run_root) + except ValueError as error: + raise OSError(f"audit artifact escapes reserved run root: {target}") from error + _atomic_write_text(target, content) + + +def _ensure_safe_audit_directory(run_root: Path, *components: str) -> Path: + """Return a checked audit directory rooted lexically beneath ``run_root``. + + Every existing component is inspected without following links. Missing + components are created one level at a time and immediately re-inspected. + Callers use the returned directory directly for the following writes; this + eliminates deterministic escapes left by a backend, but does not claim a + portable cross-process no-follow guarantee after the final inspection. + """ + + run_root = Path(run_root) + _require_safe_audit_root(run_root) + if not components: + raise ValueError("audit directory components must not be empty") + for component in components: + component_path = Path(component) + if ( + not component + or component_path.is_absolute() + or len(component_path.parts) != 1 + or component_path.name != component + or component in {".", ".."} + ): + raise OSError(f"unsafe audit directory component: {component!r}") + + directory = run_root.joinpath(*components) + try: + directory.relative_to(run_root) + except ValueError as error: + raise OSError(f"audit directory escapes reserved run root: {directory}") from error + + current = run_root + for component in components: + current = current / component + try: + metadata = os.lstat(current) + except FileNotFoundError: + try: + os.mkdir(current) + except FileExistsError: + # A racing creator must pass the same no-follow inspection. + pass + metadata = os.lstat(current) + if stat.S_ISLNK(metadata.st_mode) or _is_reparse_point(current): + raise OSError(f"refusing unsafe/reparse audit directory: {current}") + if not stat.S_ISDIR(metadata.st_mode): + raise OSError(f"audit path component is not a directory: {current}") + return directory + + +def _write_artifact_manifest( + report: OptimizationReport, + paths: RunArtifactPaths, + *, + expected_journal: str | None = None, +) -> None: + _require_mutable_temp_run(paths) + artifacts = _manifest_artifact_layout(report) + expected_files = ( + {"writeback_journal.json": expected_journal.encode("utf-8")} + if expected_journal is not None + else {} + ) + files = _manifest_file_records(paths.temp_run_dir, expected_files=expected_files) + declared_paths = _declared_manifest_paths(artifacts) + if expected_journal is not None: + declared_paths -= {"optimization_report.json", "optimization_report.md"} + available_paths = {item["path"] for item in files} + missing = declared_paths - available_paths + if missing: + raise OSError(f"audit artifact manifest is incomplete: missing {sorted(missing)}") + manifest = { + "schema_version": "eval_optimize_loop.artifacts.v1", + "run_id": paths.run_id, + "artifacts": artifacts, + "files": files, + } + _write_safe_audit_text( + paths.temp_run_dir, + "artifact_manifest.json", + _json_text(manifest), + ) + + +def _manifest_artifact_layout(report: OptimizationReport) -> dict[str, Any]: + validate_unique_round_ids(report.rounds) + candidates: list[dict[str, Any]] = [] + for index, record in enumerate(report.candidates, start=1): + candidate: CandidatePrompt = record["candidate"] + artifact_id = _candidate_artifact_name(index, candidate.candidate_id) + prompt_bundle = record.get("prompt_bundle") or candidate.bundle() + prompt_paths = { + field_name: ( + Path("candidate_prompts") + / artifact_id + / f"{_safe_artifact_name(str(field_name))}.txt" + ).as_posix() + for field_name in prompt_bundle + } + case_results: dict[str, str] = {} + for result_name in ("train_result", "validation_result"): + result = record.get(result_name) + if not isinstance(result, EvalResult): + continue + split = _safe_artifact_name(str(result.split)) + case_results[str(result.split)] = ( + Path("case_results") / f"{artifact_id}_{split}.json" + ).as_posix() + candidates.append( + { + "candidate_id": candidate.candidate_id, + "artifact_id": artifact_id, + "prompt_bundle": prompt_paths, + "diff": (Path("prompt_diffs") / f"{artifact_id}.diff").as_posix(), + "case_results": case_results, + "evaluation_failure": "evaluation_error" in record, + } + ) + + return { + "reports": { + "prewrite_json": "pre_write_report.json", + "prewrite_markdown": "pre_write_report.md", + "final_json": "optimization_report.json", + "final_markdown": "optimization_report.md", + }, + "baseline_case_results": { + "train": "case_results/baseline_train.json", + "validation": "case_results/baseline_validation.json", + }, + "baseline_prompts": { + field_name: ( + Path("baseline_prompts") / f"{_safe_artifact_name(str(field_name))}.txt" + ).as_posix() + for field_name in report.baseline_prompts + }, + "rounds": [ + (Path("rounds") / f"{_safe_artifact_name(str(round_record.round_id))}.json").as_posix() + for round_record in report.rounds + ], + "audit_records": { + "audit": "audit.json", + "config_snapshot": "config.snapshot.json", + "input_hashes": "input_hashes.json", + "rounds": "rounds.json", + "per_case_deltas": "per_case_deltas.json", + "gate_decisions": "gate_decisions.json", + "writeback": "writeback.json", + "writeback_journal": "writeback_journal.json", + "evaluation_failures": "evaluation_failures.json", + }, + "candidates": candidates, + } + + +def _declared_manifest_paths(artifacts: dict[str, Any]) -> set[str]: + declared: set[str] = set() + for group_name in ("reports", "baseline_case_results", "baseline_prompts", "audit_records"): + declared.update(artifacts[group_name].values()) + declared.update(artifacts["rounds"]) + for candidate in artifacts["candidates"]: + declared.add(candidate["diff"]) + declared.update(candidate["prompt_bundle"].values()) + declared.update(candidate["case_results"].values()) + return declared + + +def _manifest_file_records( + run_dir: Path, + *, + expected_files: dict[str, bytes], +) -> list[dict[str, Any]]: + content_by_path: dict[str, bytes] = dict(expected_files) + for artifact in run_dir.rglob("*"): + if artifact.is_symlink() or _is_reparse_point(artifact): + raise OSError(f"refusing unsafe/reparse audit artifact: {artifact}") + if not artifact.is_file(): + continue + relative_path = artifact.relative_to(run_dir).as_posix() + if relative_path == "artifact_manifest.json": + continue + content_by_path[relative_path] = artifact.read_bytes() + return [ + { + "path": relative_path, + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + } + for relative_path, content in sorted(content_by_path.items()) + ] def _candidate_artifact_name(index: int, candidate_id: str) -> str: diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py index 9741ad47..752b40fc 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -215,6 +215,7 @@ class OptimizationReport: writeback: WritebackResult = field( default_factory=lambda: WritebackResult(status="not_requested") ) + baseline_prompts: dict[str, str] = field(default_factory=dict) def to_jsonable(value: Any) -> Any: diff --git a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py index e44c9633..c96bafa3 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py +++ b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py @@ -399,6 +399,31 @@ def test_optimization_report_preserves_legacy_construction_with_new_defaults(): assert report.cost_summary == schemas.CostSummary() assert report.writeback == schemas.WritebackResult(status="not_requested") + legacy_rounds = [object()] + legacy_cost = schemas.CostSummary(total=0.5, complete=False) + legacy_writeback = schemas.WritebackResult(status="rejected") + positional_report = schemas.OptimizationReport( + "1", + {}, + {}, + empty_eval, + empty_eval, + [], + {}, + [], + {}, + [], + None, + {}, + legacy_rounds, + legacy_cost, + legacy_writeback, + ) + assert positional_report.rounds is legacy_rounds + assert positional_report.cost_summary is legacy_cost + assert positional_report.writeback is legacy_writeback + assert positional_report.baseline_prompts == {} + def test_validate_inputs_rejects_same_train_val_path(tmp_path: Path): eval_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py index 7abd87d3..912db48a 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py @@ -192,7 +192,11 @@ async def test_execute_pipeline_uses_train_only_evidence_and_full_candidate_reev assert [(call[0], *call[1:3]) for call in backend.calls] == [ ("evaluate", "baseline", "train"), ("evaluate", "baseline", "validation"), - ("optimize", {"system_prompt": "baseline\n"}, tmp_path / "out" / "runs" / "ordered_run" / "optimizer"), + ( + "optimize", + {"system_prompt": "baseline\n"}, + tmp_path / "out" / "runs" / ".ordered_run.tmp" / "optimizer", + ), ("evaluate", "candidate_a", "train"), ("evaluate", "candidate_a", "validation"), ("evaluate", "candidate_b", "train"), @@ -624,13 +628,16 @@ async def test_writeback_happens_only_after_prewrite_audit_succeeds( request = _request(tmp_path, update_source=True) backend = _safe_backend() events: list[str] = [] - artifact_paths = object() + artifact_paths: report_module.RunArtifactPaths | None = None - def prepare(report, output_dir): + def prepare(report, paths): + nonlocal artifact_paths events.append("prepare") assert report.selected_candidate == "candidate_a" - assert Path(output_dir) == tmp_path / "out" - return artifact_paths + assert paths.output_dir == tmp_path / "out" + assert paths.temp_run_dir == tmp_path / "out" / "runs" / ".ordered_run.tmp" + artifact_paths = paths + return paths def commit(snapshot, prompts): events.append("commit") @@ -641,10 +648,11 @@ def commit(snapshot, prompts): after_hashes=snapshot.hashes(), ) - def finalize(report, paths): + def finalize(report, paths, *, before_publish): events.append("finalize") assert paths is artifact_paths assert report.writeback.status == "applied" + before_publish() def persist_journal(paths, journal): events.append(f"journal:{journal['state']}") @@ -905,7 +913,7 @@ async def test_finalize_failure_leaves_explicit_prepared_writeback_journal( request = _request(tmp_path, update_source=True) backend = _safe_backend(candidate_count=1) - def fail_finalize(report, paths): + def fail_finalize(report, paths, *, before_publish): raise OSError("final report unavailable") monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", fail_finalize) @@ -913,7 +921,7 @@ def fail_finalize(report, paths): with pytest.raises(OSError, match="final report unavailable"): await execute_pipeline(request, evaluator=backend, optimizer=backend) - run_dir = Path(request.output_dir) / "runs" / request.run_id + run_dir = Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" prewrite = json.loads((run_dir / "pre_write_report.json").read_text(encoding="utf-8")) journal = json.loads((run_dir / "writeback_journal.json").read_text(encoding="utf-8")) assert prewrite["audit"]["writeback_journal"]["state"] == "pending" @@ -923,6 +931,7 @@ def fail_finalize(report, paths): assert journal["after_hashes"] == { "system_prompt": hashlib.sha256(b"safe prompt").hexdigest(), } + assert not (Path(request.output_dir) / "runs" / request.run_id).exists() @pytest.mark.asyncio @@ -993,7 +1002,9 @@ def fail_outcome(report, paths): with pytest.raises(OSError, match="convenience artifact"): await execute_pipeline(request, evaluator=backend, optimizer=backend) - journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" + journal_path = ( + Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "writeback_journal.json" + ) journal = json.loads(journal_path.read_text(encoding="utf-8")) assert (tmp_path / "prompt.txt").read_bytes() == b"safe prompt" assert journal["state"] == "applied" @@ -1012,7 +1023,12 @@ async def test_writeback_journal_is_committing_before_source_commit( real_commit = pipeline_module.commit_prompt_bundle def commit(snapshot, prompts): - journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" + journal_path = ( + Path(request.output_dir) + / "runs" + / f".{request.run_id}.tmp" + / "writeback_journal.json" + ) journal = json.loads(journal_path.read_text(encoding="utf-8")) assert journal["state"] == "committing" return real_commit(snapshot, prompts) @@ -1398,7 +1414,11 @@ async def test_candidate_prompts_must_be_valid_utf8_before_evaluation(tmp_path: assert [call[:3] for call in backend.calls] == [ ("evaluate", "baseline", "train"), ("evaluate", "baseline", "validation"), - ("optimize", {"system_prompt": "baseline\n"}, Path(request.output_dir) / "runs" / request.run_id / "optimizer"), + ( + "optimize", + {"system_prompt": "baseline\n"}, + Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "optimizer", + ), ] @@ -1789,14 +1809,17 @@ async def test_late_no_write_source_mutation_downgrades_journal_to_unknown( backend = _safe_backend(candidate_count=1) original_finalize = pipeline_module.finalize_run_artifacts - def mutate_after_finalize(report, paths): - original_finalize(report, paths) - Path(request.target_prompt_paths["system_prompt"]).write_text( - "mutation after final artifacts", - encoding="utf-8", - ) + def mutate_after_final_writes(report, paths, *, before_publish): + def mutate_then_verify(): + Path(request.target_prompt_paths["system_prompt"]).write_text( + "mutation after final artifacts", + encoding="utf-8", + ) + before_publish() + + original_finalize(report, paths, before_publish=mutate_then_verify) - monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", mutate_after_finalize) + monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", mutate_after_final_writes) with pytest.raises(ConcurrentPromptUpdateError, match="source prompt integrity changed"): await execute_pipeline(request, evaluator=backend, optimizer=backend) @@ -1804,7 +1827,7 @@ def mutate_after_finalize(report, paths): journal_path = ( Path(request.output_dir) / "runs" - / request.run_id + / f".{request.run_id}.tmp" / "writeback_journal.json" ) journal = json.loads(journal_path.read_text(encoding="utf-8")) @@ -1812,6 +1835,7 @@ def mutate_after_finalize(report, paths): assert journal["after_hashes"] == { "system_prompt": hashlib.sha256(b"mutation after final artifacts").hexdigest() } + assert not (Path(request.output_dir) / "runs" / request.run_id).exists() @pytest.mark.asyncio diff --git a/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py b/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py new file mode 100644 index 00000000..937f3b43 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py @@ -0,0 +1,722 @@ +from __future__ import annotations + +import hashlib +import json +import math +import os +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.eval_loop import report as report_module +from examples.optimization.eval_optimize_loop.eval_loop.pipeline import execute_pipeline +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary +from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationRound +from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import ( + FailingCandidateBackend, +) +from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import ( + RecordingBackend, +) +from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import _request +from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import _safe_backend + + +class _BaselineCrashBackend(RecordingBackend): + async def evaluate(self, **kwargs: Any): + artifact_dir = Path(kwargs["artifact_dir"]) + artifact_dir.joinpath("backend-started.txt").write_text("started\n", encoding="utf-8") + if kwargs["prompt_id"] == "baseline" and kwargs["split"] == "train": + raise RuntimeError("simulated backend crash") + return await super().evaluate(**kwargs) + + +def _crashing_backend() -> _BaselineCrashBackend: + delegate = _safe_backend(candidate_count=1) + return _BaselineCrashBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + + +def _temp_run_dir(request: Any) -> Path: + return Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" + + +def _final_run_dir(request: Any) -> Path: + return Path(request.output_dir) / "runs" / request.run_id + + +def _round(round_id: int, *, rationale: str) -> OptimizationRound: + return OptimizationRound( + round_id=round_id, + candidate_id="candidate_a", + prompts={"system_prompt": "safe prompt"}, + rationale=rationale, + metrics={"train_score": 1.0}, + cost=CostSummary(), + duration_seconds=0.01, + ) + + +def _assert_manifest_records_match_files(run_dir: Path, manifest: dict[str, Any]) -> None: + seen: set[str] = set() + for record in manifest["files"]: + relative_path = record["path"] + assert relative_path not in seen + seen.add(relative_path) + content = (run_dir / relative_path).read_bytes() + assert record["sha256"] == hashlib.sha256(content).hexdigest(), relative_path + assert record["size_bytes"] == len(content), relative_path + + +def _create_directory_link(link: Path, target: Path) -> None: + try: + os.symlink(target, link, target_is_directory=True) + except (NotImplementedError, OSError) as symlink_error: + pytest.skip(f"directory symlinks unavailable: {symlink_error}") + + +def _remove_directory_link(link: Path) -> None: + if link.is_symlink(): + link.unlink() + elif os.path.lexists(link): + os.rmdir(link) + + +async def _source_report(tmp_path: Path, label: str): + source_root = tmp_path / label + source_root.mkdir() + request = _request(source_root) + backend = _safe_backend(candidate_count=1) + return await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +class _FakeRenameAt2: + def __init__(self, result: int) -> None: + self.result = result + self.calls: list[tuple[Any, ...]] = [] + self.argtypes: list[Any] | None = None + self.restype: Any = None + + def __call__(self, *args: Any) -> int: + self.calls.append(args) + return self.result + + +def _install_fake_renameat2( + monkeypatch: pytest.MonkeyPatch, + *, + result: int, + error_number: int = 0, +) -> _FakeRenameAt2: + real_ctypes = report_module.ctypes + function = _FakeRenameAt2(result) + + class _FakeLibc: + renameat2 = function + + class _CtypesProxy: + def CDLL(self, name: Any, *, use_errno: bool): + assert name is None + assert use_errno is True + return _FakeLibc() + + def get_errno(self) -> int: + return error_number + + def __getattr__(self, name: str): + return getattr(real_ctypes, name) + + monkeypatch.setattr(report_module, "ctypes", _CtypesProxy()) + return function + + +def test_posix_rename_noreplace_calls_linux_abi_on_success( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + function = _install_fake_renameat2(monkeypatch, result=0) + source = tmp_path / "source" + target = tmp_path / "target" + + report_module._posix_rename_noreplace(source, target) + + assert function.argtypes == [ + report_module.ctypes.c_int, + report_module.ctypes.c_char_p, + report_module.ctypes.c_int, + report_module.ctypes.c_char_p, + report_module.ctypes.c_uint, + ] + assert function.restype is report_module.ctypes.c_int + assert function.calls == [ + (-100, os.fsencode(source), -100, os.fsencode(target), 0x1), + ] + + +def test_posix_rename_noreplace_maps_collision_to_file_exists( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + _install_fake_renameat2( + monkeypatch, + result=-1, + error_number=report_module.errno.EEXIST, + ) + target = tmp_path / "target" + + with pytest.raises(FileExistsError) as caught: + report_module._posix_rename_noreplace(tmp_path / "source", target) + + assert caught.value.errno == report_module.errno.EEXIST + assert caught.value.filename == str(target) + + +@pytest.mark.parametrize("failure", ["missing_symbol", "unsupported_kernel"]) +def test_posix_rename_noreplace_fails_closed_when_unsupported( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: str, +): + real_ctypes = report_module.ctypes + if failure == "missing_symbol": + class _LibcWithoutRenameAt2: + pass + + class _MissingSymbolProxy: + def CDLL(self, name: Any, *, use_errno: bool): + return _LibcWithoutRenameAt2() + + def __getattr__(self, name: str): + return getattr(real_ctypes, name) + + monkeypatch.setattr(report_module, "ctypes", _MissingSymbolProxy()) + expected_errno = report_module.errno.ENOTSUP + else: + _install_fake_renameat2( + monkeypatch, + result=-1, + error_number=report_module.errno.ENOSYS, + ) + expected_errno = report_module.errno.ENOSYS + + with pytest.raises(OSError) as caught: + report_module._posix_rename_noreplace(tmp_path / "source", tmp_path / "target") + + assert not isinstance(caught.value, FileExistsError) + assert caught.value.errno == expected_errno + + +def test_posix_publish_race_never_replaces_new_final_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + paths = report_module.reserve_run_artifacts(tmp_path / "out", run_id="racing_run") + evidence = paths.temp_run_dir / "evidence.txt" + evidence.write_bytes(b"reserved temp evidence") + raced_identity: dict[str, int] = {} + + def race_at_atomic_noreplace(source: Path, target: Path) -> None: + assert Path(source) == paths.temp_run_dir + assert Path(target) == paths.final_run_dir + assert not target.exists() + target.mkdir() + raced_identity["inode"] = target.stat().st_ino + raise FileExistsError(17, "final appeared during atomic publish", str(target)) + + real_os = report_module.os + + class _PosixOSProxy: + name = "posix" + + def __getattr__(self, name: str): + return getattr(real_os, name) + + monkeypatch.setattr(report_module, "os", _PosixOSProxy()) + monkeypatch.setattr(report_module, "_fsync_directory", lambda directory: None) + monkeypatch.setattr( + report_module, + "_posix_rename_noreplace", + race_at_atomic_noreplace, + raising=False, + ) + + with pytest.raises(FileExistsError, match="final appeared|already published"): + report_module._durable_publish_directory(paths) + + assert paths.final_run_dir.is_dir() + assert paths.final_run_dir.stat().st_ino == raced_identity["inode"] + assert paths.temp_run_dir.is_dir() + assert evidence.read_bytes() == b"reserved temp evidence" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reserved_name", ["ordered_run", ".ordered_run.tmp"]) +async def test_duplicate_run_id_never_overwrites_final_or_reuses_temp( + tmp_path: Path, + reserved_name: str, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + occupied = Path(request.output_dir) / "runs" / reserved_name + occupied.mkdir(parents=True) + sentinel = occupied / "sentinel.txt" + sentinel.write_bytes(b"keep me exactly") + + with pytest.raises(FileExistsError, match=request.run_id): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert sentinel.read_bytes() == b"keep me exactly" + assert backend.calls == [] + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + assert list(tmp_path.glob(".*.snapshot-*")) == [] + + +@pytest.mark.asyncio +async def test_pipeline_rejects_duplicate_round_ids_before_report_artifact_writes(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + backend.rounds = [_round(1, rationale="first"), _round(1, rationale="duplicate")] + + with pytest.raises(ValueError, match=r"duplicate round_id.*1"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + temp_run = _temp_run_dir(request) + assert temp_run.is_dir() + assert not _final_run_dir(request).exists() + for report_artifact in ( + "pre_write_report.json", + "artifact_manifest.json", + "rounds.json", + "rounds", + ): + assert not (temp_run / report_artifact).exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("writer", ["prepare", "write_reports"]) +async def test_direct_report_writers_reject_duplicate_round_ids_before_writing( + tmp_path: Path, + writer: str, +): + source_root = tmp_path / f"source-{writer}" + source_root.mkdir() + source_request = _request(source_root) + source_report = await execute_pipeline( + source_request, + evaluator=_safe_backend(candidate_count=1), + optimizer=_safe_backend(candidate_count=1), + ) + run_id = f"duplicate_rounds_{writer}" + report = replace( + source_report, + run={**source_report.run, "run_id": run_id}, + rounds=[_round(7, rationale="first"), _round(7, rationale="duplicate")], + ) + output_dir = tmp_path / f"direct-{writer}" + + if writer == "prepare": + paths = report_module.reserve_run_artifacts(output_dir, run_id=run_id) + operation = lambda: report_module.prepare_run_artifacts(report, paths) + else: + operation = lambda: report_module.write_reports(report, output_dir) + + with pytest.raises(ValueError, match=r"duplicate round_id.*7"): + operation() + + temp_run = output_dir / "runs" / f".{run_id}.tmp" + assert temp_run.is_dir() + assert list(temp_run.iterdir()) == [] + assert not (output_dir / "runs" / run_id).exists() + + +@pytest.mark.asyncio +async def test_all_runtime_backend_artifacts_use_reserved_temp_until_publication(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + temp_run = _temp_run_dir(request) + + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + artifact_dirs = [call[4] for call in backend.calls if call[0] == "evaluate"] + artifact_dirs.extend(call[2] for call in backend.calls if call[0] == "optimize") + assert artifact_dirs + assert all(path.is_relative_to(temp_run) for path in artifact_dirs) + assert not temp_run.exists() + assert _final_run_dir(request).is_dir() + + +@pytest.mark.asyncio +async def test_backend_crash_leaves_temp_evidence_without_final_publication(tmp_path: Path): + request = _request(tmp_path) + backend = _crashing_backend() + + with pytest.raises(RuntimeError, match="simulated backend crash"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + temp_run = _temp_run_dir(request) + assert (temp_run / "evaluations" / "000_baseline" / "train" / "backend-started.txt").is_file() + assert not _final_run_dir(request).exists() + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +async def test_final_artifact_write_failure_cannot_publish_a_partial_run( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + original_write = report_module._atomic_write_text + + def fail_final_markdown(path: Path, content: str) -> None: + path = Path(path) + if path.name == "optimization_report.md" and path.parent == _temp_run_dir(request): + raise OSError("simulated final report write failure") + original_write(path, content) + + monkeypatch.setattr(report_module, "_atomic_write_text", fail_final_markdown) + + with pytest.raises(OSError, match="simulated final report write failure"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert _temp_run_dir(request).is_dir() + assert (_temp_run_dir(request) / "optimization_report.json").is_file() + assert not _final_run_dir(request).exists() + assert not (Path(request.output_dir) / "optimization_report.json").exists() + assert not (Path(request.output_dir) / "optimization_report.md").exists() + + +@pytest.mark.asyncio +async def test_before_publish_failure_refreshes_manifest_without_replacing_original_error( + tmp_path: Path, +): + source_root = tmp_path / "callback-source" + source_root.mkdir() + source_request = _request(source_root) + source_report = await execute_pipeline( + source_request, + evaluator=_safe_backend(candidate_count=1), + optimizer=_safe_backend(candidate_count=1), + ) + run_id = "callback_failure" + journal = { + **source_report.audit["writeback_journal"], + "run_id": run_id, + "state": "not_requested", + } + report = replace( + source_report, + run={**source_report.run, "run_id": run_id}, + audit={**source_report.audit, "writeback_journal": journal}, + ) + paths = report_module.reserve_run_artifacts(tmp_path / "callback-output", run_id=run_id) + report_module.prepare_run_artifacts(report, paths) + + class _PromptIntegritySentinel(RuntimeError): + pass + + def mutate_journal_then_fail() -> None: + report_module.persist_writeback_journal( + paths, + { + **journal, + "state": "unknown", + "error": "source prompt integrity changed before publication", + "after_hashes": {"system_prompt": "changed"}, + }, + ) + raise _PromptIntegritySentinel("prompt drift sentinel") + + with pytest.raises(_PromptIntegritySentinel, match="prompt drift sentinel") as caught: + report_module.finalize_run_artifacts( + report, + paths, + before_publish=mutate_journal_then_fail, + ) + + assert type(caught.value) is _PromptIntegritySentinel + assert paths.temp_run_dir.is_dir() + assert not paths.final_run_dir.exists() + manifest = json.loads( + (paths.temp_run_dir / "artifact_manifest.json").read_text(encoding="utf-8") + ) + _assert_manifest_records_match_files(paths.temp_run_dir, manifest) + journal_record = next( + record for record in manifest["files"] if record["path"] == "writeback_journal.json" + ) + journal_bytes = (paths.temp_run_dir / "writeback_journal.json").read_bytes() + assert journal_record["sha256"] == hashlib.sha256(journal_bytes).hexdigest() + assert journal_record["size_bytes"] == len(journal_bytes) + + +@pytest.mark.asyncio +async def test_prepare_rejects_linked_top_level_audit_directory_before_outside_write( + tmp_path: Path, +): + source_report = await _source_report(tmp_path, "linked-top-source") + run_id = "linked_top_level" + report = replace(source_report, run={**source_report.run, "run_id": run_id}) + paths = report_module.reserve_run_artifacts(tmp_path / "linked-top-output", run_id=run_id) + outside = tmp_path / "linked-top-outside" + outside.mkdir() + sentinel = outside / "sentinel.txt" + sentinel.write_bytes(b"outside must remain untouched") + linked_component = paths.temp_run_dir / "case_results" + _create_directory_link(linked_component, outside) + + try: + with pytest.raises(OSError, match="unsafe|reparse|directory|artifact"): + report_module.prepare_run_artifacts(report, paths) + finally: + _remove_directory_link(linked_component) + + assert sentinel.read_bytes() == b"outside must remain untouched" + assert {item.name for item in outside.iterdir()} == {"sentinel.txt"} + assert paths.temp_run_dir.is_dir() + assert not paths.final_run_dir.exists() + + +@pytest.mark.asyncio +async def test_prepare_rejects_linked_candidate_directory_before_outside_write(tmp_path: Path): + source_report = await _source_report(tmp_path, "linked-candidate-source") + run_id = "linked_candidate" + report = replace(source_report, run={**source_report.run, "run_id": run_id}) + paths = report_module.reserve_run_artifacts( + tmp_path / "linked-candidate-output", + run_id=run_id, + ) + outside = tmp_path / "linked-candidate-outside" + outside.mkdir() + sentinel = outside / "sentinel.txt" + sentinel.write_bytes(b"candidate outside must remain untouched") + prompt_root = paths.temp_run_dir / "candidate_prompts" + prompt_root.mkdir() + candidate_artifact = report.audit["candidate_artifacts"]["candidate_a"] + linked_candidate = prompt_root / candidate_artifact + _create_directory_link(linked_candidate, outside) + + try: + with pytest.raises(OSError, match="unsafe|reparse|directory|artifact"): + report_module.prepare_run_artifacts(report, paths) + finally: + _remove_directory_link(linked_candidate) + + assert sentinel.read_bytes() == b"candidate outside must remain untouched" + assert {item.name for item in outside.iterdir()} == {"sentinel.txt"} + assert paths.temp_run_dir.is_dir() + assert not paths.final_run_dir.exists() + + +@pytest.mark.asyncio +async def test_prepare_rejects_non_directory_audit_component_without_replacing_it(tmp_path: Path): + source_report = await _source_report(tmp_path, "audit-file-source") + run_id = "audit_component_file" + report = replace(source_report, run={**source_report.run, "run_id": run_id}) + paths = report_module.reserve_run_artifacts(tmp_path / "audit-file-output", run_id=run_id) + component = paths.temp_run_dir / "case_results" + component.write_bytes(b"backend-owned non-directory") + + with pytest.raises(OSError): + report_module.prepare_run_artifacts(report, paths) + + assert component.read_bytes() == b"backend-owned non-directory" + assert not paths.final_run_dir.exists() + + +@pytest.mark.asyncio +async def test_each_nested_audit_write_rechecks_parent_before_next_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + source_report = await _source_report(tmp_path, "audit-parent-swap-source") + run_id = "audit_parent_swap" + report = replace(source_report, run={**source_report.run, "run_id": run_id}) + paths = report_module.reserve_run_artifacts(tmp_path / "audit-parent-swap-output", run_id=run_id) + results_dir = paths.temp_run_dir / "case_results" + parent_became_unsafe = False + original_write = report_module._atomic_write_text + original_is_reparse = report_module._is_reparse_point + + def tracked_write(path: Path, content: str) -> None: + nonlocal parent_became_unsafe + original_write(path, content) + if Path(path) == results_dir / "baseline_train.json": + parent_became_unsafe = True + + def dynamic_reparse(path: Path) -> bool: + if parent_became_unsafe and Path(path) == results_dir: + return True + return original_is_reparse(Path(path)) + + monkeypatch.setattr(report_module, "_atomic_write_text", tracked_write) + monkeypatch.setattr(report_module, "_is_reparse_point", dynamic_reparse) + + with pytest.raises(OSError, match="unsafe|reparse"): + report_module.prepare_run_artifacts(report, paths) + + assert (results_dir / "baseline_train.json").is_file() + assert not (results_dir / "baseline_validation.json").exists() + assert not paths.final_run_dir.exists() + + +@pytest.mark.asyncio +async def test_nan_serialization_failure_stays_in_temp_and_never_publishes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + original_build_report = pipeline_module.build_report + + def report_with_nan(**kwargs: Any): + report = original_build_report(**kwargs) + report.audit["non_finite"] = math.nan + return report + + monkeypatch.setattr(pipeline_module, "build_report", report_with_nan) + + with pytest.raises(ValueError, match="Out of range float values"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert _temp_run_dir(request).is_dir() + assert not _final_run_dir(request).exists() + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +async def test_fresh_published_run_has_positive_duration_and_only_strict_json(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.audit["duration_seconds"] > 0 + for artifact in _final_run_dir(request).rglob("*.json"): + payload = artifact.read_text(encoding="utf-8") + json.loads( + payload, + parse_constant=lambda value: (_ for _ in ()).throw( + ValueError(f"non-standard JSON constant: {value}") + ), + ) + + +@pytest.mark.asyncio +async def test_manifest_covers_complete_audit_set_and_partial_candidate_results(tmp_path: Path): + request = _request(tmp_path) + delegate = _safe_backend(candidate_count=1) + backend = FailingCandidateBackend( + fail_on=("candidate_a", "validation"), + failure=RuntimeError("validation service unavailable"), + delegate=delegate, + ) + backend.rounds = [_round(1, rationale="candidate proposal")] + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + run_dir = _final_run_dir(request) + manifest = json.loads((run_dir / "artifact_manifest.json").read_text(encoding="utf-8")) + _assert_manifest_records_match_files(run_dir, manifest) + artifacts = manifest["artifacts"] + assert manifest["schema_version"] == "eval_optimize_loop.artifacts.v1" + assert manifest["run_id"] == request.run_id + assert set(artifacts["reports"]) == {"prewrite_json", "prewrite_markdown", "final_json", "final_markdown"} + assert set(artifacts["baseline_case_results"]) == {"train", "validation"} + assert set(artifacts["baseline_prompts"]) == {"system_prompt"} + assert len(artifacts["rounds"]) == 1 + assert set(artifacts["audit_records"]) >= { + "audit", + "config_snapshot", + "input_hashes", + "rounds", + "per_case_deltas", + "gate_decisions", + "writeback", + "writeback_journal", + "evaluation_failures", + } + candidate = artifacts["candidates"][0] + assert candidate["artifact_id"] == report.audit["candidate_artifacts"]["candidate_a"] + assert set(candidate["prompt_bundle"]) == {"system_prompt"} + assert set(candidate["case_results"]) == {"train"} + assert candidate["evaluation_failure"] is True + + declared_files = {item["path"] for item in manifest["files"]} + for group in ( + artifacts["reports"], + artifacts["baseline_case_results"], + artifacts["baseline_prompts"], + artifacts["audit_records"], + ): + assert set(group.values()) <= declared_files + assert set(artifacts["rounds"]) <= declared_files + assert json.loads((run_dir / artifacts["rounds"][0]).read_text(encoding="utf-8"))["round_id"] == 1 + assert candidate["diff"] in declared_files + assert set(candidate["prompt_bundle"].values()) <= declared_files + assert set(candidate["case_results"].values()) <= declared_files + assert json.loads((run_dir / artifacts["audit_records"]["evaluation_failures"]).read_text(encoding="utf-8"))[ + "candidate_a" + ]["stage"] == "validation" + + +@pytest.mark.asyncio +async def test_convenience_reports_are_exact_copies_created_after_run_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + copy_events: list[tuple[str, bool, bool]] = [] + original_write_bytes = report_module._atomic_write_bytes + + def observe_copy(path: Path, content: bytes) -> None: + path = Path(path) + if path.parent == Path(request.output_dir) and path.name.startswith("optimization_report"): + copy_events.append( + ( + path.name, + _final_run_dir(request).is_dir(), + _temp_run_dir(request).exists(), + ) + ) + original_write_bytes(path, content) + + monkeypatch.setattr(report_module, "_atomic_write_bytes", observe_copy) + + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert copy_events == [ + ("optimization_report.json", True, False), + ("optimization_report.md", True, False), + ] + for name in ("optimization_report.json", "optimization_report.md"): + assert (Path(request.output_dir) / name).read_bytes() == (_final_run_dir(request) / name).read_bytes() + + +@pytest.mark.asyncio +async def test_runtime_input_hashes_match_exact_entry_bytes(tmp_path: Path): + request = _request(tmp_path) + exact_inputs = { + "train": b'{ "eval_cases" : [{"eval_id":"train_case","session_input":{"state":{}}}]}\r\n', + "validation": b'{"eval_cases":[{"eval_id":"validation_case","session_input":{"state":{}}}]} \n', + "optimizer": b'{\r\n "seed": 91\r\n}\r\n', + } + Path(request.train_path).write_bytes(exact_inputs["train"]) + Path(request.validation_path).write_bytes(exact_inputs["validation"]) + Path(request.optimizer_config_path).write_bytes(exact_inputs["optimizer"]) + prompt_bytes = b"baseline with exact trailing spaces \r\n" + Path(request.target_prompt_paths["system_prompt"]).write_bytes(prompt_bytes) + backend = _safe_backend(candidate_count=1) + + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + hashes = json.loads((_final_run_dir(request) / "input_hashes.json").read_text(encoding="utf-8")) + for role, content in exact_inputs.items(): + assert hashes[role] == hashlib.sha256(content).hexdigest() + assert hashes["target_prompts"]["system_prompt"] == hashlib.sha256(prompt_bytes).hexdigest() diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 565ce196..3041f767 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -1253,9 +1253,9 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo candidate_artifact = report.audit["candidate_artifacts"]["sdk_best"] assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / f"{candidate_artifact}.diff").is_file() assert calls["update_source"] is False - assert calls["output_dir"].endswith("runs\\sdk_test_run\\optimizer") or calls[ + assert calls["output_dir"].endswith("runs\\.sdk_test_run.tmp\\optimizer") or calls[ "output_dir" - ].endswith("runs/sdk_test_run/optimizer") + ].endswith("runs/.sdk_test_run.tmp/optimizer") assert calls["evaluation_count"] == 4 command = report.run["reproducibility_command"] assert "--sdk-call-agent fake_call_agent_module:call_agent" in command @@ -1377,7 +1377,7 @@ def test_run_pipeline_mode_sdk_explicit_run_id_is_stable_and_exclusive(tmp_path: sdk_call_agent="fake_call_agent_module:call_agent", run_id="valid_20260704-1.ok", ) - with pytest.raises(ValueError, match="run_id.*already exists"): + with pytest.raises(FileExistsError, match="run ID.*already reserved or published"): run_pipeline( mode="sdk", train_path=DEFAULT_TRAIN, From c9a7c8706a34d063446a6f9c3c1a6d6b808a1230 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 04:43:01 +0800 Subject: [PATCH 29/34] fix(evaluation): preserve prompt newline bytes --- tests/evaluation/test_target_prompt.py | 24 +++++++++++++++++++++ trpc_agent_sdk/evaluation/_target_prompt.py | 12 ++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/evaluation/test_target_prompt.py b/tests/evaluation/test_target_prompt.py index 8a2c7b48..6858e2e0 100644 --- a/tests/evaluation/test_target_prompt.py +++ b/tests/evaluation/test_target_prompt.py @@ -129,6 +129,30 @@ async def test_read_all_with_paths(tmp_path: Path): assert await target.read_all() == {"a": "alpha", "b": "beta"} +@pytest.mark.parametrize( + "original_bytes", + [ + "system prompt\nkeep exact bytes\n".encode("utf-8"), + "system prompt\r\nkeep exact bytes\r\n".encode("utf-8"), + "lf\ncrlf\r\nlone carriage return\rend".encode("utf-8"), + ], + ids=["lf", "crlf", "mixed"], +) +@pytest.mark.asyncio +async def test_path_read_write_round_trip_preserves_newline_bytes( + tmp_path: Path, + original_bytes: bytes, +): + prompt_path = tmp_path / "prompt.md" + prompt_path.write_bytes(original_bytes) + target = TargetPrompt().add_path("prompt", str(prompt_path)) + + snapshot = await target.read_all() + await target.write_all(snapshot) + + assert prompt_path.read_bytes() == original_bytes + + @pytest.mark.asyncio async def test_read_all_path_not_exist_raises_file_not_found(tmp_path: Path): target = TargetPrompt().add_path("missing", str(tmp_path / "ghost.md")) diff --git a/trpc_agent_sdk/evaluation/_target_prompt.py b/trpc_agent_sdk/evaluation/_target_prompt.py index da104f15..7fb6fd03 100644 --- a/trpc_agent_sdk/evaluation/_target_prompt.py +++ b/trpc_agent_sdk/evaluation/_target_prompt.py @@ -180,7 +180,7 @@ def _snapshot_path_contents(self) -> dict[str, Optional[str]]: for name, src in self._sources.items(): if isinstance(src, _PathSource): try: - snapshot[name] = Path(src.path).read_text(encoding="utf-8") + snapshot[name] = self._read_path(src.path) except FileNotFoundError: snapshot[name] = None return snapshot @@ -232,12 +232,18 @@ def _cleanup_tmp_files(self) -> None: @staticmethod def _atomic_write_path(path: str, content: str) -> None: tmp = path + ".tmp" - Path(tmp).write_text(content, encoding="utf-8") + with Path(tmp).open("w", encoding="utf-8", newline="") as prompt_file: + prompt_file.write(content) os.replace(tmp, path) + @staticmethod + def _read_path(path: str) -> str: + with Path(path).open("r", encoding="utf-8", newline="") as prompt_file: + return prompt_file.read() + async def _read_one(self, src: _Source) -> str: if isinstance(src, _PathSource): - return Path(src.path).read_text(encoding="utf-8") + return self._read_path(src.path) if isinstance(src, _CallbackSource): return await src.read_fn() raise TypeError(f"unknown source type: {type(src).__name__}") From 3360ef78f02ac5cb734b32f58e0a89b1ff526c9d Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 04:54:33 +0800 Subject: [PATCH 30/34] fix(evaluation): redact optimizer config snapshots --- tests/evaluation/test_agent_optimizer.py | 119 +++++++++++++++++- trpc_agent_sdk/evaluation/_agent_optimizer.py | 78 ++++++++++-- 2 files changed, 189 insertions(+), 8 deletions(-) diff --git a/tests/evaluation/test_agent_optimizer.py b/tests/evaluation/test_agent_optimizer.py index d7b061f7..6391d8e9 100644 --- a/tests/evaluation/test_agent_optimizer.py +++ b/tests/evaluation/test_agent_optimizer.py @@ -254,6 +254,8 @@ async def test_facade_persists_artifacts_under_output_dir(tmp_path, monkeypatch) """The facade must materialise result.json, summary.txt, rounds/*.json, baseline_prompts/, best_prompts/, config.snapshot.json and run.log under output_dir for every successful run.""" + import json + train = EvalSet(eval_set_id="train", eval_cases=[_eval_case("c1")]) val = EvalSet(eval_set_id="val", eval_cases=[_eval_case("c1")]) train_path = tmp_path / "train.json" @@ -287,7 +289,14 @@ async def fake_call_gepa(self, **kwargs): assert (output_dir / "result.json").is_file() assert (output_dir / "summary.txt").is_file() - assert (output_dir / "config.snapshot.json").is_file() + config_snapshot_path = output_dir / "config.snapshot.json" + assert config_snapshot_path.is_file() + config_snapshot_text = config_snapshot_path.read_text(encoding="utf-8") + assert "test-key" not in config_snapshot_text + config_snapshot = json.loads(config_snapshot_text) + assert config_snapshot["optimize"]["algorithm"]["reflection_lm"][ + "api_key" + ] == "" assert (output_dir / "run.log").is_file() assert (output_dir / "baseline_prompts" / "instruction.md").is_file() assert (output_dir / "best_prompts" / "instruction.md").is_file() @@ -297,6 +306,114 @@ async def fake_call_gepa(self, **kwargs): assert "SUCCEEDED" in log_line +def test_copy_config_snapshot_recursively_redacts_common_secret_keys(tmp_path): + """Config snapshots must remain useful without publishing credentials.""" + import json + + payload = { + "api_key": "api-secret", + "nested": [ + { + "TOKEN": "token-secret", + "access-token": "access-secret", + "Authorization": "Bearer auth-secret", + }, + { + "password": "password-secret", + "credentials": {"username": "alice", "password": "nested-secret"}, + "privateKey": "private-secret", + }, + { + "openai_api_key": "namespaced-api-secret", + "github_token": "namespaced-token-secret", + "aws_secret_access_key": "aws-secret", + "db_passwd": "database-secret", + }, + ], + "model_name": "gpt-4o", + "max_tokens": 128, + "token_budget": 256, + } + config_path = tmp_path / "optimizer.json" + config_path.write_text(json.dumps(payload), encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + + AgentOptimizer._copy_config_snapshot(str(config_path), str(output_dir)) + + snapshot_path = output_dir / "config.snapshot.json" + snapshot_text = snapshot_path.read_text(encoding="utf-8") + snapshot = json.loads(snapshot_text) + assert snapshot == { + "api_key": "", + "nested": [ + { + "TOKEN": "", + "access-token": "", + "Authorization": "", + }, + { + "password": "", + "credentials": "", + "privateKey": "", + }, + { + "openai_api_key": "", + "github_token": "", + "aws_secret_access_key": "", + "db_passwd": "", + }, + ], + "model_name": "gpt-4o", + "max_tokens": 128, + "token_budget": 256, + } + assert snapshot_text == ( + json.dumps( + snapshot, + ensure_ascii=False, + sort_keys=True, + indent=2, + allow_nan=False, + ) + + "\n" + ) + for secret in ( + "api-secret", + "token-secret", + "access-secret", + "auth-secret", + "password-secret", + "nested-secret", + "private-secret", + "namespaced-api-secret", + "namespaced-token-secret", + "aws-secret", + "database-secret", + ): + assert secret not in snapshot_text + + +@pytest.mark.parametrize( + "invalid_config", + [ + '{"api_key": "secret", invalid}', + '{"api_key": NaN}', + ], +) +def test_copy_config_snapshot_invalid_json_fails_closed(tmp_path, invalid_config): + """Malformed source config must never be copied verbatim as an artifact.""" + config_path = tmp_path / "optimizer.json" + config_path.write_text(invalid_config, encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + + AgentOptimizer._copy_config_snapshot(str(config_path), str(output_dir)) + + assert not (output_dir / "config.snapshot.json").exists() + assert not (output_dir / "config.snapshot.json.tmp").exists() + + @pytest.mark.asyncio async def test_facade_persists_artifacts_when_algorithm_fails(tmp_path, monkeypatch): """Even when the algorithm returns a FAILED result the facade should diff --git a/trpc_agent_sdk/evaluation/_agent_optimizer.py b/trpc_agent_sdk/evaluation/_agent_optimizer.py index 16510e75..7f907c3d 100644 --- a/trpc_agent_sdk/evaluation/_agent_optimizer.py +++ b/trpc_agent_sdk/evaluation/_agent_optimizer.py @@ -15,6 +15,7 @@ from __future__ import annotations import inspect +import json import logging import os import signal @@ -52,6 +53,56 @@ _PROMPT_FILE_LOGGER = logging.getLogger("trpc_agent_sdk.optimizer") +_REDACTED_CONFIG_VALUE = "" +_SENSITIVE_CONFIG_KEY_SUFFIXES = ( + "apikey", + "authorization", + "authtoken", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "idtoken", + "password", + "passwd", + "privatekey", + "refreshtoken", + "secret", + "secretaccesskey", + "secretkey", + "token", + "accesstoken", +) + + +def _reject_non_finite_json_constant(value: str) -> Any: + """Reject Python's non-standard JSON constants (NaN and infinities).""" + raise ValueError(f"non-finite JSON constant is not allowed: {value}") + + +def _redact_config_secrets(value: Any) -> Any: + """Return a JSON-compatible copy with credential values redacted. + + Key matching is case-insensitive, ignores separators and accepts + namespaced credential suffixes, so ``api_key``, ``API-Key`` and + ``openai_api_key`` share the same policy without redacting safe fields + such as ``max_tokens``. + """ + if isinstance(value, dict): + redacted = {} + for key, child in value.items(): + normalized_key = "".join( + character for character in key.casefold() if character.isalnum() + ) + if normalized_key.endswith(_SENSITIVE_CONFIG_KEY_SUFFIXES): + redacted[key] = _REDACTED_CONFIG_VALUE + else: + redacted[key] = _redact_config_secrets(child) + return redacted + if isinstance(value, list): + return [_redact_config_secrets(child) for child in value] + return value + def _atomic_write_text(path: str, content: str) -> None: """Atomically replace ``path`` with ``content`` (UTF-8). @@ -157,8 +208,8 @@ async def optimize( output_dir: Required artifact directory. The facade creates it when missing and persists ``result.json``, ``summary.txt``, ``rounds/`` records, ``baseline_prompts/`` and ``best_prompts/`` - directories, a ``config.snapshot.json`` copy of the input - config, and a ``run.log`` summary line. + directories, a redacted ``config.snapshot.json`` copy of the + input config, and a ``run.log`` summary line. callbacks: Optional evaluator lifecycle callbacks. update_source: When True, persist the best candidate back to every registered TargetPrompt field after a SUCCEEDED @@ -393,7 +444,7 @@ def _persist_artifacts( (regardless of update_source). - ``best_prompts/.md`` Best candidate per field (only when a result was produced). - - ``config.snapshot.json`` Copy of the input config. + - ``config.snapshot.json`` Redacted copy of the input config. - ``run.log`` One-line status footer. SIGINT (Ctrl+C) is masked for the duration of this method so a @@ -473,12 +524,25 @@ def _write_rounds_directory(result: OptimizeResult, output_dir: str) -> None: def _copy_config_snapshot(config_path: str, output_dir: str) -> None: target = os.path.join(output_dir, "config.snapshot.json") try: - # Read + atomic-write rather than shutil.copyfile so an interrupted - # copy cannot leave a half-written ``config.snapshot.json``. - content = Path(config_path).read_text(encoding="utf-8") + # Parse and redact fully before the first write so neither the + # destination nor its temporary sibling can ever contain the raw + # credential-bearing config. Invalid/non-finite JSON therefore + # fails closed instead of falling back to a plaintext copy. + raw_config = json.loads( + Path(config_path).read_text(encoding="utf-8"), + parse_constant=_reject_non_finite_json_constant, + ) + redacted_config = _redact_config_secrets(raw_config) + content = json.dumps( + redacted_config, + ensure_ascii=False, + sort_keys=True, + indent=2, + allow_nan=False, + ) + "\n" _atomic_write_text(target, content) except Exception: # pragma: no cover - _PROMPT_FILE_LOGGER.warning("failed to copy config snapshot", exc_info=True) + _PROMPT_FILE_LOGGER.warning("failed to write redacted config snapshot", exc_info=True) @staticmethod def _write_run_log(*, output_dir: str, line: str) -> None: From 41d9e89b75d87c0e837f94aaeb7b2edd23d86e61 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 08:30:05 +0800 Subject: [PATCH 31/34] fix(examples): finalize report timing semantics --- .../eval_optimize_loop/eval_loop/backends.py | 28 +++++- .../eval_optimize_loop/eval_loop/report.py | 12 +-- .../tests/test_pipeline_fake_mode.py | 18 +++- .../tests/test_sdk_backend.py | 90 +++++++++++++++++++ 4 files changed, 139 insertions(+), 9 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index 0c0c3754..afa010d1 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -5,6 +5,7 @@ import asyncio import importlib import math +import time from collections.abc import Iterable from dataclasses import dataclass from dataclasses import replace @@ -130,6 +131,7 @@ async def optimize_candidates( baseline_prompts, context="cannot optimize fake prompt bundle", ) + proposal_started_at = time.perf_counter() candidates = _normalize_fake_candidates( self._optimizer.propose( baseline_prompt, @@ -137,6 +139,13 @@ async def optimize_candidates( failure_summary, ) ) + proposal_duration_seconds = _positive_perf_duration( + proposal_started_at, + time.perf_counter(), + ) + round_duration_seconds = ( + proposal_duration_seconds / len(candidates) if candidates else 0.0 + ) zero_cost = CostSummary(complete=True) rounds = [ OptimizationRound( @@ -146,7 +155,7 @@ async def optimize_candidates( rationale=candidate.rationale, metrics={}, cost=zero_cost, - duration_seconds=0.0, + duration_seconds=round_duration_seconds, ) for index, candidate in enumerate(candidates, start=1) ] @@ -158,6 +167,10 @@ async def optimize_candidates( "backend": "fake", "baseline_prompt_id": baseline_train.prompt_id, "failure_summary": _safe_jsonable(failure_summary), + "proposal_duration_seconds": proposal_duration_seconds, + "round_duration_allocation": ( + "equal_share_of_batch_proposal_duration" + ), }, ) @@ -837,6 +850,19 @@ def _finite_number(value: Any, *, context: str) -> float: return number +def _positive_perf_duration(started_at: float, finished_at: float) -> float: + """Return one finite positive duration for a measured perf-counter span.""" + + elapsed = finished_at - started_at + if math.isfinite(elapsed) and elapsed > 0.0: + return elapsed + + resolution = float(time.get_clock_info("perf_counter").resolution) + if not math.isfinite(resolution) or resolution <= 0.0: + raise RuntimeError("perf_counter resolution must be finite and positive") + return resolution + + def _nonnegative_number(value: Any, *, context: str) -> float: number = _finite_number(value, context=context) if number < 0.0: diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index a3c18d4f..7cc63734 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -152,7 +152,7 @@ def build_report( if isinstance(result, EvalResult): all_results.append(result) return OptimizationReport( - schema_version="eval_optimize_loop.v1", + schema_version="eval_optimize_loop.v2", run=run, baseline={"train": baseline_train, "validation": baseline_validation}, baseline_train=baseline_train, @@ -645,7 +645,7 @@ def render_markdown(report: OptimizationReport) -> str: "", "## Failure Attribution Summary", "", - f"Total failed case evaluations: {summary['total_failed_cases']}", + f"Total failed case evaluations: {summary['total_failed_cases']}", "", "| category | count |", "| --- | ---: |", @@ -691,8 +691,8 @@ def render_markdown(report: OptimizationReport) -> str: candidate = record["candidate"] lines.extend([ f"### {candidate.candidate_id}", - "", - "```diff", + "", + "```diff", candidate.prompt_diff, "```", "", @@ -702,8 +702,8 @@ def render_markdown(report: OptimizationReport) -> str: "## Reproducibility", "", f"```{report.run.get('reproducibility_shell') or 'bash'}", - report.run.get("reproducibility_command") - or report.audit.get("reproducibility_command") + report.run.get("reproducibility_command") + or report.audit.get("reproducibility_command") or REPRODUCIBILITY_COMMAND, "```", "", diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py index fe7013e5..375e926c 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -6,7 +6,6 @@ import sys from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG -from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline @@ -15,11 +14,18 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): output_dir = tmp_path / "run" + prompt_path = tmp_path / "baseline_system_prompt.txt" + prompt_path.write_text( + "You are a helpful support assistant.\n\n" + "Answer clearly and include a short explanation when it may help the user.\n" + "If the user asks for structured data, provide the information they need.\n", + encoding="utf-8", + ) report = run_pipeline( train_path=DEFAULT_TRAIN, val_path=DEFAULT_VAL, optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, - prompt_path=DEFAULT_PROMPT, + prompt_path=prompt_path, output_dir=output_dir, fake_model=True, fake_judge=True, @@ -31,8 +37,10 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert json_path.is_file() assert md_path.is_file() assert report.selected_candidate == "candidate_002_safe" + assert report.schema_version == "eval_optimize_loop.v2" payload = json.loads(json_path.read_text(encoding="utf-8")) + assert payload["schema_version"] == "eval_optimize_loop.v2" assert set(payload) >= { "schema_version", "run", @@ -213,5 +221,11 @@ def _normalized_payload(payload: dict) -> dict: normalized["run"].pop("reproducibility_command", None) normalized["audit"].pop("duration_seconds", None) normalized["audit"].pop("reproducibility_command", None) + normalized["audit"].get("sdk_result_summary", {}).pop( + "proposal_duration_seconds", + None, + ) normalized["audit"].get("writeback_journal", {}).pop("run_id", None) + for round_record in normalized.get("rounds", []): + round_record.pop("duration_seconds", None) return normalized diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 3041f767..88cb8df7 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -3,6 +3,7 @@ import builtins import inspect import json +import math import os import shlex import sys @@ -127,6 +128,75 @@ async def test_fake_backend_wraps_candidates_in_complete_optimization_result(tmp assert all(candidate.bundle() == {"system_prompt": candidate.prompt} for candidate in result.candidates) +@pytest.mark.asyncio +async def test_fake_backend_distributes_measured_batch_proposal_duration( + tmp_path: Path, + monkeypatch, +): + perf_counter_values = iter((10.0, 10.006)) + monkeypatch.setattr( + backend_module, + "time", + types.SimpleNamespace( + perf_counter=lambda: next(perf_counter_values), + get_clock_info=lambda name: types.SimpleNamespace(resolution=1e-9), + ), + raising=False, + ) + + result = await backend_module.FakeBackend(seed=91).optimize_candidates( + baseline_prompts={"system_prompt": "baseline prompt\n"}, + baseline_train=_failed_fake_train_result(), + failure_summary={"by_category": {"format_violation": 1}}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "fake_optimize", + ) + + durations = [round_record.duration_seconds for round_record in result.rounds] + assert durations == pytest.approx([0.003, 0.003]) + assert all(duration > 0 and math.isfinite(duration) for duration in durations) + assert sum(durations) == pytest.approx(0.006) + assert result.raw_summary["proposal_duration_seconds"] == pytest.approx(0.006) + assert result.raw_summary["round_duration_allocation"] == ( + "equal_share_of_batch_proposal_duration" + ) + + +@pytest.mark.asyncio +async def test_fake_backend_uses_clock_resolution_when_batch_timer_does_not_advance( + tmp_path: Path, + monkeypatch, +): + perf_counter_values = iter((42.0, 42.0)) + monkeypatch.setattr( + backend_module, + "time", + types.SimpleNamespace( + perf_counter=lambda: next(perf_counter_values), + get_clock_info=lambda name: types.SimpleNamespace(resolution=1e-6), + ), + raising=False, + ) + + result = await backend_module.FakeBackend(seed=91).optimize_candidates( + baseline_prompts={"system_prompt": "baseline prompt\n"}, + baseline_train=_failed_fake_train_result(), + failure_summary={"by_category": {"format_violation": 1}}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "fake_optimize", + ) + + durations = [round_record.duration_seconds for round_record in result.rounds] + assert durations == pytest.approx([0.5e-6, 0.5e-6]) + assert all(duration > 0 and math.isfinite(duration) for duration in durations) + assert sum(durations) == pytest.approx(1e-6) + assert result.raw_summary["proposal_duration_seconds"] == pytest.approx(1e-6) + + @pytest.mark.asyncio async def test_fake_backend_returns_no_candidates_without_failure_categories(tmp_path: Path): baseline_prompt = DEFAULT_PROMPT.read_text(encoding="utf-8") @@ -1962,6 +2032,26 @@ def _empty_eval_result(prompt_id: str, split: str) -> EvalResult: ) +def _failed_fake_train_result() -> EvalResult: + return EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[ + CaseResult( + case_id="observed_training_failure", + split="train", + score=0.0, + passed=False, + output="not-json", + failure_category="format_violation", + ) + ], + ) + + def _sdk_round( round_id: int, candidate_prompts: dict[str, str], From eaed59ff91392f5229d4206daee4e762420cf9f6 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 08:31:26 +0800 Subject: [PATCH 32/34] test(examples): prove issue 91 acceptance thresholds --- .../tests/fixtures/attribution_cases.json | 10 + .../tests/fixtures/holdout_gate_cases.json | 12 + .../tests/test_acceptance_thresholds.py | 238 ++++++++++++++++++ .../tests/test_sdk_integration.py | 219 ++++++++++++++++ 4 files changed, 479 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json create mode 100644 examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json create mode 100644 examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py diff --git a/examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json b/examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json new file mode 100644 index 00000000..7cee577a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json @@ -0,0 +1,10 @@ +[ + {"error_code": "json_parse_failure", "evidence": "JSON parser failed", "expected": "format_violation"}, + {"error_code": "required_key_missing", "evidence": "missing key status", "expected": "final_response_mismatch"}, + {"error_code": "exact_answer_mismatch", "evidence": "expected YES", "expected": "final_response_mismatch"}, + {"error_code": "tool_call_error", "evidence": "expected lookup", "expected": "tool_call_error"}, + {"error_code": "parameter_error", "evidence": "id mismatch", "expected": "parameter_error"}, + {"error_code": "missing_rubric_terms", "evidence": "missing latency", "expected": "llm_rubric_not_met"}, + {"error_code": "knowledge_recall_insufficient", "evidence": "missing doc-a", "expected": "knowledge_recall_insufficient"}, + {"error_code": "forbidden_pattern", "evidence": "forbidden JSON", "expected": "format_violation"} +] diff --git a/examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json b/examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json new file mode 100644 index 00000000..0d9db73a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json @@ -0,0 +1,12 @@ +[ + {"id": "safe_gain", "expected": true, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 0.02, "budget": 1.0}, + {"id": "no_gain", "expected": false, "train": [1, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 0.02, "budget": 1.0}, + {"id": "overfit", "expected": false, "train": [0, 0, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 0, 0], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "validation_regression", "expected": false, "train": [1, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 1, 1], "candidate_validation": [1, 1, 0], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "protected_regression", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 0, 0], "candidate_validation": [0, 1, 1], "protected": ["v0"], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "new_hard_fail", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 0, 0], "candidate_validation": [0, 1, 1], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "soft_to_hard", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.5, 0, 0], "candidate_validation": [0, 1, 1], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "single_case_drop", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.7, 0, 0], "candidate_validation": [0.5, 1, 1], "protected": [], "max_drop": 0.1, "cost": 0.02, "budget": 1.0}, + {"id": "over_budget", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 1.1, "budget": 1.0}, + {"id": "safe_gain_with_soft_failure", "expected": true, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.5, 0, 1], "candidate_validation": [0.5, 1, 1], "protected": ["v0"], "max_drop": 0.0, "cost": 0.02, "budget": 1.0} +] diff --git a/examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py b/examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py new file mode 100644 index 00000000..767c8c56 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import json +import shutil +import time +from pathlib import Path +from typing import Any + +from examples.optimization.eval_optimize_loop.eval_loop.attribution import attribute_failure +from examples.optimization.eval_optimize_loop.eval_loop.gate import AcceptanceGate +from examples.optimization.eval_optimize_loop.eval_loop.report import compute_case_deltas +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import GateDecision +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_holdout_gate_fixture_has_ten_unique_scenarios() -> None: + scenarios = _load_fixture("holdout_gate_cases.json") + + assert len(scenarios) >= 10 + scenario_ids = [str(scenario["id"]) for scenario in scenarios] + assert len(scenario_ids) == len(set(scenario_ids)) + expected_labels = {bool(scenario["expected"]) for scenario in scenarios} + assert expected_labels == {False, True} + logical_inputs = [ + json.dumps( + { + key: value + for key, value in scenario.items() if key not in {"id", "expected"} + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) for scenario in scenarios + ] + assert len(logical_inputs) == len(set(logical_inputs)) + + +def test_gate_thresholds_reject_always_false_predictions() -> None: + scenarios = _load_fixture("holdout_gate_cases.json") + expected = [bool(scenario["expected"]) for scenario in scenarios] + + assert not _gate_acceptance_thresholds_met( + expected, + predictions=[False] * len(expected), + ) + + +def test_attribution_fixture_has_eight_unique_evidence_inputs() -> None: + scenarios = _load_fixture("attribution_cases.json") + + assert len(scenarios) >= 8 + evidence_inputs = [(str(scenario["error_code"]), str(scenario["evidence"])) for scenario in scenarios] + assert len(evidence_inputs) == len(set(evidence_inputs)) + error_codes = [str(scenario["error_code"]) for scenario in scenarios] + assert len(error_codes) == len(set(error_codes)) + + +def test_holdout_gate_decision_accuracy_is_at_least_eighty_percent() -> None: + scenarios = _load_fixture("holdout_gate_cases.json") + expected_labels: list[bool] = [] + predictions: list[bool] = [] + for scenario in scenarios: + expected_labels.append(bool(scenario["expected"])) + inputs = {key: value for key, value in scenario.items() if key != "expected"} + decision = _decision_for_scenario(inputs) + predictions.append(decision.accepted) + + overall_accuracy, positive_recall, negative_specificity = (_gate_classification_rates(expected_labels, predictions)) + assert overall_accuracy >= 0.80 + assert positive_recall >= 0.80 + assert negative_specificity >= 0.80 + + +def test_independent_attribution_accuracy_is_at_least_seventy_five_percent() -> None: + scenarios = _load_fixture("attribution_cases.json") + inputs = [{key: value for key, value in scenario.items() if key != "expected"} for scenario in scenarios] + predictions = [_attribute_scenario(item) for item in inputs] + + correct = sum(prediction[0] == scenario["expected"] + for prediction, scenario in zip(predictions, scenarios, strict=True)) + assert correct / len(scenarios) >= 0.75 + assert all(prediction[1] and prediction[2] for prediction in predictions) + + +def test_fake_trace_pipeline_finishes_under_three_minutes(tmp_path: Path) -> None: + inputs_dir = tmp_path / "inputs" + inputs_dir.mkdir() + train_path = inputs_dir / "train.evalset.json" + val_path = inputs_dir / "val.evalset.json" + prompt_path = inputs_dir / "baseline_system_prompt.txt" + optimizer_config_path = inputs_dir / "optimizer.json" + shutil.copyfile(DEFAULT_TRAIN, train_path) + shutil.copyfile(DEFAULT_VAL, val_path) + shutil.copyfile(DEFAULT_PROMPT, prompt_path) + optimizer_config_path.write_text( + json.dumps( + { + "seed": 91, + "optimizer": {}, + "metrics": {}, + "gate": { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, + }, + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + started = time.perf_counter() + report = run_pipeline( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + trace=True, + run_id="performance", + ) + elapsed = time.perf_counter() - started + + assert elapsed < 180 + assert 0 < report.audit["duration_seconds"] <= elapsed + + +def _load_fixture(name: str) -> list[dict[str, Any]]: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def _gate_acceptance_thresholds_met( + expected: list[bool], + predictions: list[bool], +) -> bool: + rates = _gate_classification_rates(expected, predictions) + return all(rate >= 0.80 for rate in rates) + + +def _gate_classification_rates( + expected: list[bool], + predictions: list[bool], +) -> tuple[float, float, float]: + paired = list(zip(predictions, expected, strict=True)) + positive_predictions = [prediction for prediction, label in paired if label] + negative_predictions = [prediction for prediction, label in paired if not label] + assert paired + assert positive_predictions + assert negative_predictions + overall_accuracy = sum(prediction == label for prediction, label in paired) / len(paired) + positive_recall = sum(positive_predictions) / len(positive_predictions) + negative_specificity = (sum(not prediction for prediction in negative_predictions) / len(negative_predictions)) + return overall_accuracy, positive_recall, negative_specificity + + +def _eval( + prompt_id: str, + split: str, + scores: list[float], + *, + cost: float = 0.0, +) -> EvalResult: + cases = [ + CaseResult( + case_id=f"{split[0]}{index}", + split=split, + score=float(score), + passed=score >= 1.0, + output=str(score), + metrics={"holdout": float(score)}, + hard_failed=score == 0.0, + ) for index, score in enumerate(scores) + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=sum(scores) / len(scores), + passed=all(case.passed for case in cases), + cost=cost, + cases=cases, + ) + + +def _decision_for_scenario(scenario: dict[str, Any]) -> GateDecision: + assert "expected" not in scenario + baseline_train = _eval("baseline", "train", scenario["train"]) + baseline_validation = _eval("baseline", "validation", scenario["validation"]) + candidate_train = _eval("candidate", "train", scenario["candidate_train"]) + candidate_validation = _eval( + "candidate", + "validation", + scenario["candidate_validation"], + ) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + ) + gate = AcceptanceGate({ + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": scenario["protected"], + "max_score_drop_per_case": scenario["max_drop"], + "max_total_cost": scenario["budget"], + }) + incurred_cost = float(scenario["cost"]) + return gate.decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + deltas=deltas, + cumulative_cost=incurred_cost, + cost_summary=CostSummary( + optimizer=incurred_cost, + total=incurred_cost, + complete=True, + ), + ) + + +def _attribute_scenario(scenario: dict[str, Any]) -> tuple[str, str, str]: + assert "expected" not in scenario + return attribute_failure(scenario["error_code"], scenario["evidence"]) diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py new file mode 100644 index 00000000..4b83c957 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.eval_loop.schemas import to_jsonable +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline_async +from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation import AgentOptimizer +from trpc_agent_sdk.evaluation import GepaReflectiveOptimizer +from trpc_agent_sdk.evaluation import TargetPrompt + + +class FakeGEPAResult: + + def __init__( + self, + baseline: dict[str, str], + candidate: dict[str, str], + ) -> None: + self.candidates = [baseline, candidate] + self.val_aggregate_scores = [2 / 3, 1.0] + self.parents = [[None], [0]] + self.discovery_eval_counts = [0, 1] + self.total_metric_calls = 6 + self.best_outputs_valset = None + self.per_objective_best_candidates = {} + + @property + def best_idx(self) -> int: + return 1 + + +@pytest.mark.asyncio +async def test_sdk_pipeline_uses_real_facade_evaluator_and_post_gate_writeback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inputs_dir = tmp_path / "inputs" + inputs_dir.mkdir() + train_path = inputs_dir / "train.evalset.json" + val_path = inputs_dir / "val.evalset.json" + prompt_path = inputs_dir / "system_prompt.txt" + optimizer_config_path = inputs_dir / "optimizer.json" + gate_path = inputs_dir / "gate.json" + shutil.copyfile(DEFAULT_TRAIN, train_path) + shutil.copyfile(DEFAULT_VAL, val_path) + expected_train_case_ids = _eval_ids(train_path) + expected_val_case_ids = _eval_ids(val_path) + shutil.copyfile(DEFAULT_PROMPT, prompt_path) + optimizer_config_path.write_text( + json.dumps( + { + "evaluate": { + "metrics": [{ + "metric_name": "final_response_avg_score", + "threshold": 0.5, + }], + "num_runs": 1, + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": None + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 91, + "reflection_lm": { + "provider_name": "openai", + "model_name": "gpt-4o", + "api_key": "test-key", + }, + "max_metric_calls": 6, + }, + }, + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + gate_path.write_text( + json.dumps( + { + "gate": { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 0.0, + "max_total_cost": None, + } + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + baseline = {"system_prompt": prompt_path.read_text(encoding="utf-8")} + candidate = { + "system_prompt": + (baseline["system_prompt"].rstrip("\n") + "\n\nUse strict JSON only when the user explicitly asks.\n") + } + gepa_calls: list[dict[str, object]] = [] + + async def fake_call_gepa( + self: GepaReflectiveOptimizer, + **kwargs: object, + ) -> FakeGEPAResult: + assert isinstance(self, GepaReflectiveOptimizer) + gepa_calls.append(dict(kwargs)) + return FakeGEPAResult(baseline, candidate) + + async def call_agent(query: str) -> str: + prompt = prompt_path.read_text(encoding="utf-8") + return deterministic_sdk_response(prompt, query) + + monkeypatch.setattr( + GepaReflectiveOptimizer, + "_call_gepa_optimize", + fake_call_gepa, + ) + module = ModuleType("issue91_sdk_call_agent") + module.call_agent = call_agent + monkeypatch.setitem(sys.modules, module.__name__, module) + + report = await run_pipeline_async( + mode="sdk", + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + sdk_call_agent="issue91_sdk_call_agent:call_agent", + gate_config_path=gate_path, + update_source=True, + run_id="sdk-integration", + ) + + assert AgentOptimizer.__module__.startswith("trpc_agent_sdk.evaluation.") + assert AgentEvaluator.__module__.startswith("trpc_agent_sdk.evaluation.") + assert TargetPrompt.__module__.startswith("trpc_agent_sdk.evaluation.") + assert GepaReflectiveOptimizer.__module__.startswith("trpc_agent_sdk.evaluation.") + assert len(gepa_calls) == 1 + gepa_call = gepa_calls[0] + assert gepa_call["seed_candidate"] == baseline + captured_trainset = gepa_call["trainset"] + captured_valset = gepa_call["valset"] + assert isinstance(captured_trainset, list) + assert isinstance(captured_valset, list) + assert {case.eval_id for case in captured_trainset} == expected_train_case_ids + assert {case.eval_id for case in captured_valset} == expected_val_case_ids + assert gepa_call["callbacks"] + assert report.selected_candidate == "sdk_round_001" + assert report.candidates + for record in report.candidates: + assert {case.case_id for case in record["train_result"].cases} == expected_train_case_ids + assert {case.case_id for case in record["validation_result"].cases} == expected_val_case_ids + assert report.gate_decisions + assert all(item.gate_status == "applied" for item in report.gate_decisions) + decision = next(item for item in report.gate_decisions if item.candidate_id == "sdk_round_001") + assert decision.gate_status == "applied" + assert decision.accepted is True + assert report.writeback.status == "applied" + assert prompt_path.read_text(encoding="utf-8") == candidate["system_prompt"] + + sdk_summary = report.audit["sdk_result_summary"] + assert sdk_summary["status"] == "SUCCEEDED" + assert sdk_summary["extras"]["total_metric_calls"] == 6 + assert sdk_summary["total_llm_cost"] == 0.0 + assert report.cost_summary.reported_optimizer_cost == 0.0 + assert "partial_applied" not in json.dumps(to_jsonable(report)) + run_dir = tmp_path / "out" / "runs" / "sdk-integration" + assert (run_dir / "optimizer" / "result.json").is_file() + optimizer_config_snapshot = run_dir / "optimizer" / "config.snapshot.json" + artifact_files = sorted(path for path in run_dir.rglob("*") if path.is_file()) + assert optimizer_config_snapshot in artifact_files + leaked_test_key = [ + path.relative_to(run_dir).as_posix() for path in artifact_files if b"test-key" in path.read_bytes() + ] + assert leaked_test_key == [] + + +def deterministic_sdk_response(prompt: str, query: str) -> str: + case = EvalCase( + case_id="sdk-runtime-query", + split="runtime", + input=query, + expectation={"type": "runtime-only; must not be inspected"}, + ) + output, _, _ = FakeModel(seed=91).generate( + "sdk-runtime", + prompt, + case.input, + ) + try: + parsed = json.loads(output) + except json.JSONDecodeError: + return output + return json.dumps( + parsed, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def _eval_ids(path: Path) -> set[str]: + payload = json.loads(path.read_text(encoding="utf-8")) + return {str(case["eval_id"]) for case in payload["eval_cases"]} From 092fb40cf4f39b9611716b26dcbb747391f87704 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 11:25:39 +0800 Subject: [PATCH 33/34] docs(examples): finalize issue 91 closed loop --- .../optimization/eval_optimize_loop/DESIGN.md | 15 +- .../optimization/eval_optimize_loop/README.md | 134 +-- .../eval_optimize_loop/data/optimizer.json | 60 +- .../eval_optimize_loop/eval_loop/artifacts.py | 35 +- .../eval_loop/attribution.py | 12 +- .../eval_optimize_loop/eval_loop/backends.py | 344 +++---- .../eval_optimize_loop/eval_loop/config.py | 38 +- .../eval_optimize_loop/eval_loop/evaluator.py | 3 +- .../eval_loop/fake_judge.py | 59 +- .../eval_loop/fake_model.py | 3 +- .../eval_optimize_loop/eval_loop/gate.py | 61 +- .../eval_optimize_loop/eval_loop/optimizer.py | 42 +- .../eval_optimize_loop/eval_loop/pipeline.py | 301 +++--- .../eval_optimize_loop/eval_loop/report.py | 193 ++-- .../eval_optimize_loop/eval_loop/schemas.py | 7 +- .../eval_optimize_loop/eval_loop/writeback.py | 22 +- .../outputs/optimization_report.example.json | 857 ++++++++++-------- .../outputs/optimization_report.example.md | 29 +- .../eval_optimize_loop/run_pipeline.py | 52 +- .../tests/test_config_validation.py | 49 +- .../tests/test_failure_attribution.py | 24 +- .../tests/test_fake_model_generalization.py | 196 ++-- .../eval_optimize_loop/tests/test_gate.py | 44 +- .../test_no_sample_case_id_hardcoding.py | 4 +- .../tests/test_pipeline_fake_mode.py | 18 +- .../tests/test_pipeline_orchestration.py | 380 ++++---- .../tests/test_report_artifacts.py | 121 ++- .../tests/test_sdk_backend.py | 232 +++-- .../tests/test_writeback.py | 111 ++- trpc_agent_sdk/evaluation/_agent_optimizer.py | 4 +- 30 files changed, 1739 insertions(+), 1711 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 42a0e80a..ac11c3f7 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -1,16 +1,9 @@ # 设计说明 -## 失败归因 -评测先由 FakeJudge 产出结构化失败，再用规则归因：JSON、禁用格式归为 `format_violation`，精确答案错归为 `final_response_mismatch`，工具名、参数、知识召回、长度和 rubric 各有独立类别。每个失败都保留 reason 与 evidence，便于复核。若样例声明期望类别，报告会计算归因准确率。 +本示例把基线评测、失败归因、提示词优化、候选复评、接受门禁和审计落盘串成同一条可复现流水线。训练集与验证集采用 SDK 标准 EvalSet；FakeModel 只读取提示词和用户输入，不能接触期望答案、标签、受保护标记或样例编号，真值仅供评测器、归因器和门禁使用，从源头避免答案泄漏。 -## 接受门禁 -Gate 要求验证集提升达到阈值，并检查新硬失败、受保护样例退化、单例降分和累计成本。若训练分上涨但验证不涨，直接标记过拟合并拒绝。 +失败结果按格式、最终回复、工具调用、参数、知识召回和规则评分等类别聚合，每个失败样例保留原因与证据。训练集只用于暴露问题，所有去重后的候选都必须重新执行完整训练集和验证集评测，再逐样例计算新增通过、新增失败、升分和降分。统一门禁检查验证集增益、过拟合、新增硬失败、受保护样例退化、单例降分和累计成本；总成本不完整时，任何数值预算都以 `cost_unavailable` 拒绝。 -## 防过拟合 -训练集只用于暴露问题，候选必须通过验证集和 protected case。过拟合候选即使修好训练格式，只要把验证集自然语言或受保护精确答案改坏，也会被拒绝。`delta_type` 标出 new_pass、new_fail、score_up、score_down，避免只看平均分掩盖局部退化。 +SDK 优化始终使用 `update_source=False`，并记录真实逐样例指标、失败证据及轨迹是否可用。只有候选通过完整门禁、输入完整性检查且预写审计成功后，`--update-source` 才由共享流水线事务性回写所选提示词；失败时保留原始字节或明确报告恢复错误。 -## Fake 与 SDK -Fake mode 由 expectation、tags、protected 和 simulated_outputs 驱动，不依赖样例 id，可无 API key 稳定复现。SDK mode 通过 `SDKBackend` 调用 `AgentOptimizer` 与 `TargetPrompt`，失败时给出明确错误，不回退到 fake。 - -## 审计与回写 -报告同时写 JSON、Markdown 和 `runs//`，保存输入哈希、配置快照、候选 prompt、diff、case 结果与成本，并给出可复现命令。默认不回写源 prompt，只有显式 `--update-source` 才允许，报告会记录该选择。 +运行产物先写入 `runs/..tmp/`，包含基线与候选提示词、每轮记录、全部分割评测结果、逐样例差异、门禁、成本和回写日志。发布时采用不可覆盖的原子目录重命名：Windows 使用 MoveFileExW，Linux 使用 renameat2；其他缺少安全原语的平台直接失败。最终目录不可变，清单记录每个文件的 SHA-256 与大小，顶层报告仅是发布后的便利副本。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index fa541c8a..89992154 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -17,7 +17,7 @@ loader.py / config.py --> validated cases, gate config, input hashes v backends.py |-- FakeBackend -> FakeModel + FakeJudge + FakeOptimizer - `-- SDKBackend -> AgentOptimizer + TargetPrompt + user call_agent + `-- SDKBackend -> AgentOptimizer + TargetPrompt + user call_agent + AgentEvaluator | v attribution.py -> evaluator.py -> gate.py -> report.py @@ -87,47 +87,59 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ needed by that callable. SDK mode never silently falls back to fake mode. The generated reproducibility command records the actual `--sdk-call-agent` `module:function` target and file/config paths, but it does not record API keys -or other provider secrets. +or other provider secrets. Persisted optimizer config snapshots recursively +replace credential values with ``. + +The checked-in `train.evalset.json` and `val.evalset.json` use the SDK +`EvalSet` schema (`eval_set_id` / `eval_cases`). Fake mode reads its +deterministic judge metadata from each case's `session_input.state` +(`eval_optimize_expectation`, `eval_optimize_tags`, and +`eval_optimize_protected`). SDK optimizer config and wrapper gate config are intentionally separate. `--optimizer-config` is passed unchanged to `AgentOptimizer.optimize`, so it must follow the SDK `OptimizeConfigFile` schema. Put wrapper-only gate settings in `--gate-config` (for example `{"gate": {"min_val_score_improvement": 0.05, -"max_total_cost": 1.0}}`). If `--gate-config` is omitted, the wrapper uses the -same default aggregate gate values as the fake example. +"allow_new_hard_fail": false, "max_score_drop_per_case": 0.0, +"max_total_cost": null}}`). If `--gate-config` is omitted, the wrapper uses the +same default gate values as fake mode and auto-adds validation cases marked with +`eval_optimize_protected`. A numeric `max_total_cost` fails closed with +`cost_unavailable` whenever the backend cannot prove a complete total. Use +`null` only when you intentionally want to run the quality gates without a cost +limit; `reported_optimizer_cost` remains an incomplete lower bound, not a total. `--target-prompt name=path` may be repeated. If omitted, SDK mode keeps the old single-field behavior and registers `system_prompt=--prompt`. A run can optimize only `router_prompt`, only `skill_prompt`, or any set of named fields as long as `OptimizeResult.best_prompts` returns every registered field. -Fake mode is the complete per-case closed loop. SDK mode is the real -`AgentOptimizer` / `TargetPrompt` path with an aggregate wrapper gate. When the -SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult` -aggregate fields into the JSON/Markdown report: baseline/best pass rate, -pass-rate improvement, metric breakdowns, token usage, duration, LLM cost, -all `best_prompts`, and round summaries. SDK mode applies -`gate_status: partial_applied`: it checks `status == SUCCEEDED`, validation -improvement against `gate.min_val_score_improvement`, and total LLM cost against -`gate.max_total_cost`. Protected-case regression, new-hard-failure, per-case -delta, and per-case score-drop checks are not claimed in SDK mode unless the SDK -exposes full per-case validation scores; they are listed in -`not_applied_checks`. - -Fake mode uses a deterministic run id (`eval_optimize_loop_seed_`) so the -example outputs are byte-stable. SDK mode is append-only by default: the wrapper -derives a compact UTC `run.run_id` from the SDK result `started_at` when -available, otherwise from the current UTC timestamp. Pass `--run-id` only when -a fixed audit path is useful for tests or local smoke runs; only explicit -`--run-id` values are included in the reproducibility command. +Fake mode is the deterministic offline closed loop. SDK mode is the real +`AgentOptimizer` / `TargetPrompt` path followed by explicit post-optimization +`AgentEvaluator` runs. The adapter deduplicates every prompt bundle found in +`OptimizeResult.rounds[].candidate_prompts` and `best_prompts`. The shared +pipeline temporarily installs each bundle, reruns the complete train and +validation evalsets, restores the exact source bytes, then computes the same +per-case deltas and complete gate decision used by fake mode. SDK case results +carry real per-metric scores, failure reason/evidence, and `trace_available`; +missing SDK trace data is reported as unavailable rather than fabricated. +Optimizer aggregates remain audit context and are never substituted for these +post-optimization case results. + +Both modes default to a collision-resistant run id of the form +`eval_optimize_loop___`. Pass `--run-id` +when a stable path is useful. Existing temporary or final run ids are never +reused or overwritten. The checked-in example is generated in a fresh output +directory with the explicit id `example`. ## Source Prompt Writes -The default is **no source write-back**. The baseline prompt file is not modified -by fake mode, and `SDKBackend` calls `AgentOptimizer.optimize(update_source=False)` -unless `--update-source` is explicitly passed. The report records -`run.update_source` and the Markdown report states whether source write-back was -enabled. +The default is **no source write-back**. `SDKBackend` always calls +`AgentOptimizer.optimize(update_source=False)` so the optimizer can never commit +early. `--update-source` is handled only by the shared pipeline: after all +candidate evaluations, full gate acceptance, input-integrity checks, and audit +preparation, it transactionally writes the selected prompt bundle. This rule is +the same in fake and SDK modes. The report records the requested action and the +terminal `writeback` state. ## Candidate Behavior @@ -139,49 +151,57 @@ The fake optimizer proposes exactly two candidates: requested; it improves validation without protected-case regression, so the gate may accept it. -The fake model is driven by `EvalCase.expectation`, `tags`, `protected`, and -optional `simulated_outputs`; it does not depend on sample `case_id` names. +The fake model reads only the prompt and user input. It cannot inspect +expectations, labels, tags, protected markers, or case ids. Those fields are +available only to the fake judge, failure attribution, and acceptance gate. ## Reports -`optimization_report.json` includes: +`optimization_report.json` uses `eval_optimize_loop.v2` and includes: - `schema_version`; - `run` metadata: mode, fake flags, trace flag, case counts, update-source flag, and input paths; - `baseline` plus compatibility fields `baseline_train` and `baseline_validation`; -- all candidate train/validation results, rationale, and prompt diff; +- `baseline_prompts`, all candidate prompt bundles, complete train/validation + results, rationale, and prompt diff; +- optimizer `rounds`, `cost_summary`, and terminal `writeback`; - per-case deltas with `delta_type` (`new_pass`, `new_fail`, `score_up`, `score_down`, `unchanged`); - failure attribution summary and attribution accuracy when expected labels are present; - gate decisions with overfit detection, protected regressions, new hard - failures, excessive drops, cost fields, and SDK `not_applied_checks` when - per-case validation details are not exposed; -- audit data: seed, duration, config hash, input hashes, candidate prompt hashes, - cost, prompt diffs, and reproducibility command. + failures, excessive drops, and cost fields in both fake and SDK modes; +- audit data: seed, real duration, redacted config and raw-file hashes, input + hashes, candidate artifact mapping, complete/incomplete cost semantics, + prompt integrity journal, and reproducibility command. -`gate.max_total_cost` is interpreted as the total evaluated run cost at the time -each candidate is judged: baseline cost plus all candidates evaluated so far, -including rejected candidates. This makes budget decisions deterministic and -auditable when multiple candidates are considered. +When `CostSummary.complete` is true, `gate.max_total_cost` is interpreted as the +total evaluated run cost when each candidate is judged: baseline cost plus all +candidates evaluated so far, including rejected candidates. When it is false, +the report separates `known_run_cost` and `reported_optimizer_cost`, and any +numeric total-cost gate rejects with `cost_unavailable`. `optimization_report.md` includes final decision, gate reasons, score table, per-case delta table, failure attribution summary, cost/audit details, prompt diffs, and the reproducibility command. -`report.py` also writes audit artifacts under `output_dir/runs//`: - -- `config.snapshot.json`; -- `input_hashes.json` with train, validation, optimizer, prompt, - `target_prompts.`, and optional `gate_config` hashes; -- fake mode: `candidate_prompts//system_prompt.txt`; -- SDK mode: `candidate_prompts//.txt` for every - returned `best_prompts` field, plus `bundle.txt` with the combined prompt - shown in the wrapper report; -- `case_results/_.json`; -- `prompt_diffs/.diff`. +Audit data is first written under `output_dir/runs/..tmp/`. After every +final artifact and prompt-integrity check succeeds, the directory is published +once to `output_dir/runs//` with no-replace semantics. Windows uses +`MoveFileExW` without a replace flag; Linux uses +`renameat2(RENAME_NOREPLACE)`. A POSIX platform without that atomic primitive +fails closed. Published run directories are authoritative and immutable; the +top-level JSON/Markdown files are convenience copies created afterward. + +Each run contains `pre_write_report.*`, final reports, a redacted config +snapshot, input hashes, `baseline_prompts/`, every candidate prompt field and +diff, baseline and candidate split results, per-round JSON, per-case deltas, +gate decisions, evaluation failures, `writeback.json`, and the durable +writeback journal. SDK optimizer artifacts are kept beneath `optimizer/` with a +redacted config snapshot. `artifact_manifest.json` maps the logical artifacts +and records the SHA-256 and byte size of every file. The repository keeps only stable examples: @@ -197,9 +217,9 @@ directories are not committed. python -m pytest examples/optimization/eval_optimize_loop/tests ``` -The tests cover fake hidden-sample generalization, config validation, gate -rejection paths, protected-case behavior, failure attribution, tool/knowledge -judge paths, SDK adapter wiring through monkeypatching, deterministic report -generation, and both CLI forms. CI can run fake mode plus the monkeypatched SDK -smoke tests without real API credentials; real SDK/model calls are opt-in local -or integration runs. +The tests cover fake hidden-sample generalization, SDK EvalSet compatibility, +config validation, gate rejection paths, protected-case behavior, failure +attribution, tool/knowledge judge paths, SDK adapter and post-evaluator wiring +through monkeypatching, deterministic report generation, and both CLI forms. CI +can run fake mode plus the monkeypatched SDK smoke tests without real API +credentials; real SDK/model calls are opt-in local or integration runs. diff --git a/examples/optimization/eval_optimize_loop/data/optimizer.json b/examples/optimization/eval_optimize_loop/data/optimizer.json index dcd4c57a..5886597c 100644 --- a/examples/optimization/eval_optimize_loop/data/optimizer.json +++ b/examples/optimization/eval_optimize_loop/data/optimizer.json @@ -1,20 +1,48 @@ { - "seed": 91, - "optimizer": { - "name": "fake_two_candidate_optimizer", - "description": "Deterministically proposes one overfit candidate and one safe candidate." - }, - "metrics": { - "case_score": "mean", - "failure_attribution": "rule_based" - }, - "gate": { - "min_val_score_improvement": 0.01, - "allow_new_hard_fail": false, - "protected_case_ids": [ - "val_protected_yes_no" + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + } ], - "max_score_drop_per_case": 0.0, - "max_total_cost": 1.0 + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 91, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 2048, + "temperature": 0.4 + } + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "frontier_type": "instance", + "reflection_minibatch_size": 3, + "reflection_history_top_k": 2, + "skip_perfect_score": false, + "use_merge": false, + "max_metric_calls": 18, + "score_threshold": 1.0, + "max_iterations_without_improvement": 3 + } } } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py b/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py index 8a08b4f0..11e02b45 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import Any - ARTIFACT_COMPONENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") WINDOWS_RESERVED_NAMES = { "CON", @@ -23,13 +22,8 @@ def validate_artifact_component(value: Any, *, context: str) -> str: """Return a portable artifact component or fail before filesystem access.""" - if ( - not isinstance(value, str) - or value in {"", ".", ".."} - or len(value) > MAX_ARTIFACT_COMPONENT_LENGTH - or value.endswith((".", " ")) - or not ARTIFACT_COMPONENT_RE.fullmatch(value) - ): + if (not isinstance(value, str) or value in {"", ".", ".."} or len(value) > MAX_ARTIFACT_COMPONENT_LENGTH + or value.endswith((".", " ")) or not ARTIFACT_COMPONENT_RE.fullmatch(value)): raise ValueError(f"unsafe {context}: {value!r}") windows_stem = value.split(".", 1)[0].upper() if windows_stem in WINDOWS_RESERVED_NAMES: @@ -57,10 +51,8 @@ def validate_distinct_file_paths( portable_key = str(resolved).casefold() previous_label = resolved_keys.get(portable_key) if previous_label is not None: - raise ValueError( - f"{context} must be different physical files; " - f"{previous_label!r} and {label!r} collide case-insensitively" - ) + raise ValueError(f"{context} must be different physical files; " + f"{previous_label!r} and {label!r} collide case-insensitively") try: metadata = path.stat() @@ -69,16 +61,11 @@ def validate_distinct_file_paths( except OSError as error: raise ValueError(f"{context} {label!r} is unavailable: {error}") from error - physical_key = ( - (int(metadata.st_dev), int(metadata.st_ino)) - if metadata is not None and int(metadata.st_ino) != 0 - else None - ) + physical_key = ((int(metadata.st_dev), + int(metadata.st_ino)) if metadata is not None and int(metadata.st_ino) != 0 else None) if physical_key is not None and physical_key in physical_keys: - raise ValueError( - f"{context} must be different physical files; " - f"{physical_keys[physical_key]!r} and {label!r} are aliases" - ) + raise ValueError(f"{context} must be different physical files; " + f"{physical_keys[physical_key]!r} and {label!r} are aliases") for observed_label, observed_path, observed_exists in observed_paths: if metadata is None or not observed_exists: @@ -90,10 +77,8 @@ def validate_distinct_file_paths( except OSError as error: raise ValueError(f"{context} {label!r} is unavailable: {error}") from error if same_file: - raise ValueError( - f"{context} must be different physical files; " - f"{observed_label!r} and {label!r} are aliases" - ) + raise ValueError(f"{context} must be different physical files; " + f"{observed_label!r} and {label!r} are aliases") resolved_keys[portable_key] = label if physical_key is not None: diff --git a/examples/optimization/eval_optimize_loop/eval_loop/attribution.py b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py index 53818346..2b298bc8 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/attribution.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py @@ -7,7 +7,6 @@ from .schemas import EvalResult - _ERROR_TO_ATTRIBUTION = { "json_parse_failure": ("format_violation", "output is not valid JSON"), "required_key_missing": ("final_response_mismatch", "required JSON key is missing"), @@ -71,12 +70,11 @@ def summarize_failures(results: Iterable[EvalResult]) -> dict[str, object]: return { "total_failed_cases": total_failed, "by_category": dict(sorted(by_category.items())), - "by_prompt_split": {key: dict(sorted(value.items())) for key, value in sorted(by_prompt.items())}, + "by_prompt_split": { + key: dict(sorted(value.items())) + for key, value in sorted(by_prompt.items()) + }, "examples": examples, - "attribution_accuracy": ( - round(expected_correct / expected_total, 6) - if expected_total - else None - ), + "attribution_accuracy": (round(expected_correct / expected_total, 6) if expected_total else None), "expected_labeled_failures": expected_total, } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py index afa010d1..e8ad751b 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/backends.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -106,13 +106,11 @@ async def evaluate( for case_result in result.cases: model_trace = case_result.trace.get("model") sanitized_trace = dict(model_trace) if isinstance(model_trace, dict) else {} - sanitized_cases.append( - replace( - case_result, - trace=sanitized_trace, - trace_available=bool(sanitized_trace), - ) - ) + sanitized_cases.append(replace( + case_result, + trace=sanitized_trace, + trace_available=bool(sanitized_trace), + )) return replace(result, cases=sanitized_cases) async def optimize_candidates( @@ -137,15 +135,12 @@ async def optimize_candidates( baseline_prompt, baseline_train, failure_summary, - ) - ) + )) proposal_duration_seconds = _positive_perf_duration( proposal_started_at, time.perf_counter(), ) - round_duration_seconds = ( - proposal_duration_seconds / len(candidates) if candidates else 0.0 - ) + round_duration_seconds = (proposal_duration_seconds / len(candidates) if candidates else 0.0) zero_cost = CostSummary(complete=True) rounds = [ OptimizationRound( @@ -156,8 +151,7 @@ async def optimize_candidates( metrics={}, cost=zero_cost, duration_seconds=round_duration_seconds, - ) - for index, candidate in enumerate(candidates, start=1) + ) for index, candidate in enumerate(candidates, start=1) ] return OptimizationResult( candidates=candidates, @@ -168,9 +162,7 @@ async def optimize_candidates( "baseline_prompt_id": baseline_train.prompt_id, "failure_summary": _safe_jsonable(failure_summary), "proposal_duration_seconds": proposal_duration_seconds, - "round_duration_allocation": ( - "equal_share_of_batch_proposal_duration" - ), + "round_duration_allocation": ("equal_share_of_batch_proposal_duration"), }, ) @@ -186,11 +178,9 @@ def optimize( """Reject legacy optimization that has no observed failure evidence.""" del baseline_prompt, train_path, val_path, optimizer_config_path, output_dir - raise RuntimeError( - "FakeBackend.optimize() cannot invent training failure evidence; " - "use await FakeBackend.optimize_candidates(...) with baseline_train " - "and failure_summary." - ) + raise RuntimeError("FakeBackend.optimize() cannot invent training failure evidence; " + "use await FakeBackend.optimize_candidates(...) with baseline_train " + "and failure_summary.") class SDKBackend: @@ -232,10 +222,8 @@ def optimize( """Safely bridge old synchronous callers to the async implementation.""" if _has_running_loop(): - raise ValueError( - "SDKBackend.optimize() cannot be called while an event loop is already running; " - "use await SDKBackend.optimize_async(...) instead." - ) + raise ValueError("SDKBackend.optimize() cannot be called while an event loop is already running; " + "use await SDKBackend.optimize_async(...) instead.") return asyncio.run( self.optimize_async( baseline_prompt=baseline_prompt, @@ -243,8 +231,7 @@ def optimize( val_path=val_path, optimizer_config_path=optimizer_config_path, output_dir=output_dir, - ) - ) + )) async def optimize_async( self, @@ -309,14 +296,10 @@ async def optimize_candidates( target_paths, context="baseline prompt bundle", ) - mismatched = sorted( - name for name in target_paths if baseline_bundle[name] != source_bundle[name] - ) + mismatched = sorted(name for name in target_paths if baseline_bundle[name] != source_bundle[name]) if mismatched: - raise ValueError( - "baseline prompt bundle does not match registered source prompt files: " - + ", ".join(mismatched) - ) + fields = ", ".join(mismatched) + raise ValueError(f"baseline prompt bundle does not match registered source prompt files: {fields}") target_prompt = TargetPrompt() for name, path in target_paths.items(): @@ -336,10 +319,8 @@ async def optimize_candidates( finally: changed_sources = _changed_snapshot_files(snapshot) if changed_sources: - raise RuntimeError( - "AgentOptimizer modified source prompt files despite update_source=False: " - + ", ".join(changed_sources) - ) + fields = ", ".join(changed_sources) + raise RuntimeError(f"AgentOptimizer modified source prompt files despite update_source=False: {fields}") _require_successful_optimize_result(sdk_result) total_llm_cost = _nonnegative_result_field( @@ -382,15 +363,11 @@ async def optimize_candidates( seen_round_ids.add(round_id) candidate_id = f"sdk_round_{round_id:03d}" round_raw_prompts = getattr(sdk_round, "candidate_prompts", {}) or {} - round_prompts = ( - _validated_prompt_bundle( - round_raw_prompts, - target_paths, - context=f"SDK round {round_id} candidate_prompts", - ) - if round_raw_prompts - else {} - ) + round_prompts = (_validated_prompt_bundle( + round_raw_prompts, + target_paths, + context=f"SDK round {round_id} candidate_prompts", + ) if round_raw_prompts else {}) rationale = str(getattr(sdk_round, "acceptance_reason", "") or "") round_metrics = _finite_metric_map( getattr(sdk_round, "metric_breakdown", {}) or {}, @@ -425,8 +402,7 @@ async def optimize_candidates( complete=False, ), duration_seconds=duration_seconds, - ) - ) + )) if round_prompts: bundle_key = _prompt_bundle_key(round_prompts) if bundle_key not in seen_bundles: @@ -437,8 +413,7 @@ async def optimize_candidates( prompts=round_prompts, rationale=rationale, baseline_prompts=baseline_bundle, - ) - ) + )) best_key = _prompt_bundle_key(best_prompts) if best_key not in seen_bundles: @@ -448,8 +423,7 @@ async def optimize_candidates( prompts=best_prompts, rationale="Best prompt returned by AgentOptimizer.optimize.", baseline_prompts=baseline_bundle, - ) - ) + )) raw_summary = _summarize_sdk_result(sdk_result) cost = CostSummary( @@ -501,9 +475,9 @@ async def evaluate( artifact_path.mkdir(parents=True, exist_ok=True) eval_config_path = artifact_path / "eval_config.json" eval_config_path.write_text( - EvalConfig( - criteria={"final_response_avg_score": 1.0} - ).model_dump_json(indent=2), + EvalConfig(criteria={ + "final_response_avg_score": 1.0 + }).model_dump_json(indent=2), encoding="utf-8", ) snapshot = snapshot_prompt_files(target_paths) @@ -540,17 +514,12 @@ def _load_required_call_agent(self, *, for_evaluation: bool): suffix = " for AgentEvaluator runs" if for_evaluation else ( ". The callable must be async and compatible with " "AgentOptimizer.optimize(call_agent=...). Also configure real model credentials required " - "by that callable, such as TRPC_AGENT_API_KEY/TRPC_AGENT_BASE_URL/TRPC_AGENT_MODEL_NAME." - ) + "by that callable, such as TRPC_AGENT_API_KEY/TRPC_AGENT_BASE_URL/TRPC_AGENT_MODEL_NAME.") raise ValueError(f"sdk mode requires --sdk-call-agent module:function{suffix}") return _load_call_agent(self.call_agent_path) def _target_prompt_paths(self) -> dict[str, str | Path]: - paths = ( - dict(self.target_prompt_paths) - if self.target_prompt_paths - else {"system_prompt": self.prompt_path} - ) + paths = (dict(self.target_prompt_paths) if self.target_prompt_paths else {"system_prompt": self.prompt_path}) validate_distinct_file_paths(paths, context="SDK target prompt fields") return paths @@ -575,8 +544,7 @@ def _normalize_fake_candidates(candidates: Iterable[CandidatePrompt]) -> list[Ca rationale=candidate.rationale, prompt_diff=candidate.prompt_diff, prompt_fields=bundle, - ) - ) + )) return normalized @@ -615,25 +583,19 @@ def _validated_prompt_bundle( missing_fields = sorted(name for name in paths if name not in prompts) if missing_fields: if context == "OptimizeResult.best_prompts": - raise ValueError( - "sdk mode completed but OptimizeResult.best_prompts is missing registered target fields: " - + ", ".join(missing_fields) - ) + fields = ", ".join(missing_fields) + raise ValueError("sdk mode completed but OptimizeResult.best_prompts is missing " + f"registered target fields: {fields}") raise ValueError(f"{context} is missing registered target fields: {', '.join(missing_fields)}") extra_fields = sorted(name for name in prompts if name not in paths) if extra_fields: raise ValueError(f"{context} contains unregistered target fields: {', '.join(extra_fields)}") - empty_fields = sorted( - name - for name in paths - if not isinstance(prompts[name], str) or not prompts[name].strip() - ) + empty_fields = sorted(name for name in paths if not isinstance(prompts[name], str) or not prompts[name].strip()) if empty_fields: if context == "OptimizeResult.best_prompts": - raise ValueError( - "sdk mode completed but OptimizeResult.best_prompts contained empty registered target fields: " - + ", ".join(empty_fields) - ) + fields = ", ".join(empty_fields) + raise ValueError("sdk mode completed but OptimizeResult.best_prompts contained " + f"empty registered target fields: {fields}") raise ValueError(f"{context} contains empty registered target fields: {', '.join(empty_fields)}") return {name: prompts[name] for name in paths} @@ -645,9 +607,7 @@ def _read_prompt_bundle(paths: dict[str, str | Path]) -> dict[str, str]: try: prompts[name] = prompt_path.read_bytes().decode("utf-8") except UnicodeDecodeError as exc: - raise ValueError( - f"source prompt field {name!r} is not valid UTF-8: {prompt_path}" - ) from exc + raise ValueError(f"source prompt field {name!r} is not valid UTF-8: {prompt_path}") from exc return prompts @@ -657,9 +617,7 @@ def _prompt_bundle_from_snapshot(snapshot: Any) -> dict[str, str]: try: prompts[name] = prompt_file.content.decode("utf-8") except UnicodeDecodeError as exc: - raise ValueError( - f"source prompt field {name!r} is not valid UTF-8: {prompt_file.path}" - ) from exc + raise ValueError(f"source prompt field {name!r} is not valid UTF-8: {prompt_file.path}") from exc return prompts @@ -687,13 +645,11 @@ def _require_successful_optimize_result(result: Any) -> None: status = _sdk_result_text(getattr(result, "status", None)).upper() if status == "SUCCEEDED": return - raise ValueError( - "SDK optimization did not succeed: " - f"status={status}; " - f"error_message={_sdk_result_text(getattr(result, 'error_message', None))}; " - f"finish_reason={_sdk_result_text(getattr(result, 'finish_reason', None))}; " - f"stop_reason={_sdk_result_text(getattr(result, 'stop_reason', None))}" - ) + raise ValueError("SDK optimization did not succeed: " + f"status={status}; " + f"error_message={_sdk_result_text(getattr(result, 'error_message', None))}; " + f"finish_reason={_sdk_result_text(getattr(result, 'finish_reason', None))}; " + f"stop_reason={_sdk_result_text(getattr(result, 'stop_reason', None))}") def _sdk_result_text(value: Any) -> str: @@ -733,58 +689,75 @@ def _candidate_from_bundle( def _summarize_sdk_result(result: Any) -> dict[str, Any]: return { - "schema_version": _safe_jsonable(getattr(result, "schema_version", None)), - "algorithm": _safe_jsonable(getattr(result, "algorithm", None)), - "status": _safe_jsonable(getattr(result, "status", None)), - "finish_reason": _safe_jsonable(getattr(result, "finish_reason", None)), - "stop_reason": _safe_jsonable(getattr(result, "stop_reason", None)), - "error_message": _safe_jsonable(getattr(result, "error_message", None)), - "baseline_pass_rate": _pass_rate_result_field( + "schema_version": + _safe_jsonable(getattr(result, "schema_version", None)), + "algorithm": + _safe_jsonable(getattr(result, "algorithm", None)), + "status": + _safe_jsonable(getattr(result, "status", None)), + "finish_reason": + _safe_jsonable(getattr(result, "finish_reason", None)), + "stop_reason": + _safe_jsonable(getattr(result, "stop_reason", None)), + "error_message": + _safe_jsonable(getattr(result, "error_message", None)), + "baseline_pass_rate": + _pass_rate_result_field( "baseline_pass_rate", getattr(result, "baseline_pass_rate", 0.0), ), - "best_pass_rate": _pass_rate_result_field( + "best_pass_rate": + _pass_rate_result_field( "best_pass_rate", getattr(result, "best_pass_rate", 0.0), ), - "pass_rate_improvement": _finite_result_field( + "pass_rate_improvement": + _finite_result_field( "pass_rate_improvement", getattr(result, "pass_rate_improvement", 0.0), ), - "baseline_metric_breakdown": _finite_metric_map( + "baseline_metric_breakdown": + _finite_metric_map( getattr(result, "baseline_metric_breakdown", {}) or {}, context="SDK OptimizeResult baseline_metric_breakdown", ), - "best_metric_breakdown": _finite_metric_map( + "best_metric_breakdown": + _finite_metric_map( getattr(result, "best_metric_breakdown", {}) or {}, context="SDK OptimizeResult best_metric_breakdown", ), - "metric_thresholds": _finite_metric_map( + "metric_thresholds": + _finite_metric_map( getattr(result, "metric_thresholds", {}) or {}, context="SDK OptimizeResult metric_thresholds", ), - "per_metric_best_candidates": _safe_jsonable( - getattr(result, "per_metric_best_candidates", {}) - ), - "total_llm_cost": _nonnegative_result_field( + "per_metric_best_candidates": + _safe_jsonable(getattr(result, "per_metric_best_candidates", {})), + "total_llm_cost": + _nonnegative_result_field( "total_llm_cost", getattr(result, "total_llm_cost", 0.0), ), - "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})), - "duration_seconds": _nonnegative_result_field( + "total_token_usage": + _safe_jsonable(getattr(result, "total_token_usage", {})), + "duration_seconds": + _nonnegative_result_field( "duration_seconds", getattr(result, "duration_seconds", 0.0), ), - "started_at": _safe_jsonable(getattr(result, "started_at", None)), - "finished_at": _safe_jsonable(getattr(result, "finished_at", None)), - "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)), - "baseline_prompts": _safe_jsonable(getattr(result, "baseline_prompts", {})), - "best_prompts": _safe_jsonable(getattr(result, "best_prompts", {})), - "rounds": [ - _round_raw_summary(round_record) - for round_record in getattr(result, "rounds", []) or [] - ], - "extras": _safe_jsonable(getattr(result, "extras", {})), + "started_at": + _safe_jsonable(getattr(result, "started_at", None)), + "finished_at": + _safe_jsonable(getattr(result, "finished_at", None)), + "total_rounds": + _safe_jsonable(getattr(result, "total_rounds", 0)), + "baseline_prompts": + _safe_jsonable(getattr(result, "baseline_prompts", {})), + "best_prompts": + _safe_jsonable(getattr(result, "best_prompts", {})), + "rounds": [_round_raw_summary(round_record) for round_record in getattr(result, "rounds", []) or []], + "extras": + _safe_jsonable(getattr(result, "extras", {})), } @@ -795,14 +768,10 @@ def _round_raw_summary(round_record: Any) -> dict[str, Any]: return { "round": _safe_jsonable(getattr(round_record, "round", None)), "candidate_prompts": _safe_jsonable(getattr(round_record, "candidate_prompts", {})), - "validation_pass_rate": _safe_jsonable( - getattr(round_record, "validation_pass_rate", None) - ), + "validation_pass_rate": _safe_jsonable(getattr(round_record, "validation_pass_rate", None)), "metric_breakdown": _safe_jsonable(getattr(round_record, "metric_breakdown", {})), "accepted": _safe_jsonable(getattr(round_record, "accepted", None)), - "acceptance_reason": _safe_jsonable( - getattr(round_record, "acceptance_reason", "") - ), + "acceptance_reason": _safe_jsonable(getattr(round_record, "acceptance_reason", "")), "failed_case_ids": _safe_jsonable(getattr(round_record, "failed_case_ids", [])), "round_llm_cost": _safe_jsonable(getattr(round_record, "round_llm_cost", 0.0)), "duration_seconds": _safe_jsonable(getattr(round_record, "duration_seconds", 0.0)), @@ -826,19 +795,14 @@ def _nonnegative_result_field(field_name: str, value: Any) -> float: def _pass_rate_result_field(field_name: str, value: Any) -> float: number = _finite_result_field(field_name, value) if not 0.0 <= number <= 1.0: - raise ValueError( - f"SDK OptimizeResult field {field_name} must be between 0 and 1" - ) + raise ValueError(f"SDK OptimizeResult field {field_name} must be between 0 and 1") return number def _finite_metric_map(value: Any, *, context: str) -> dict[str, float]: if not isinstance(value, dict): raise ValueError(f"{context} must be a metric mapping") - return { - str(name): _finite_number(score, context=f"{context}.{name}") - for name, score in value.items() - } + return {str(name): _finite_number(score, context=f"{context}.{name}") for name, score in value.items()} def _finite_number(value: Any, *, context: str) -> float: @@ -923,14 +887,10 @@ def _load_sdk_expected_cases( expected_cases: dict[str, EvalCase] = {} for index, raw_case in enumerate(raw_cases): if not isinstance(raw_case, dict): - raise ValueError( - f"SDK evalset {dataset_path} eval_cases[{index}] must be an object" - ) + raise ValueError(f"SDK evalset {dataset_path} eval_cases[{index}] must be an object") eval_id = raw_case.get("eval_id") if not isinstance(eval_id, str) or not eval_id.strip(): - raise ValueError( - f"SDK evalset {dataset_path} eval_cases[{index}] is missing non-empty eval_id" - ) + raise ValueError(f"SDK evalset {dataset_path} eval_cases[{index}] is missing non-empty eval_id") if eval_id in expected_cases: raise ValueError(f"SDK evalset {dataset_path} contains duplicate eval_id {eval_id!r}") expected_cases[eval_id] = _expected_case_from_sdk_case( @@ -962,9 +922,7 @@ def _expected_case_from_sdk_case( if user_content is None: continue if not isinstance(user_content, dict): - raise ValueError( - f"{context} conversation[{turn_index}].user_content must be an object" - ) + raise ValueError(f"{context} conversation[{turn_index}].user_content must be an object") candidate_text = _content_text(user_content) if candidate_text.strip(): input_text = candidate_text @@ -980,9 +938,7 @@ def _expected_case_from_sdk_case( raise ValueError(f"{context} session_input.state must be an object") expectation = state.get("eval_optimize_expectation") if not isinstance(expectation, dict): - raise ValueError( - f"{context} session_input.state must contain eval_optimize_expectation object" - ) + raise ValueError(f"{context} session_input.state must contain eval_optimize_expectation object") tags = state.get("eval_optimize_tags", []) if not isinstance(tags, list) or any(not isinstance(tag, str) for tag in tags): @@ -996,13 +952,8 @@ def _expected_case_from_sdk_case( expected_failure_category = state.get("expected_failure_category") if expected_failure_category is None: expected_failure_category = expectation.get("expected_failure_category") - if ( - expected_failure_category is not None - and ( - not isinstance(expected_failure_category, str) - or not expected_failure_category.strip() - ) - ): + if (expected_failure_category is not None + and (not isinstance(expected_failure_category, str) or not expected_failure_category.strip())): raise ValueError(f"{context} expected_failure_category must be a non-empty string") return EvalCase( @@ -1059,10 +1010,8 @@ def _eval_result_from_sdk_result( raise ValueError(f"SDK evaluation result contains duplicate case id: {eval_id}") run_list = list(runs or []) if num_runs is not None and len(run_list) != num_runs: - raise ValueError( - f"SDK eval set {eval_set_id!r} declares num_runs={num_runs}, " - f"but case {eval_id!r} contains {len(run_list)} runs" - ) + raise ValueError(f"SDK eval set {eval_set_id!r} declares num_runs={num_runs}, " + f"but case {eval_id!r} contains {len(run_list)} runs") _validate_sdk_run_ids( run_list, eval_set_id=eval_set_id, @@ -1091,38 +1040,19 @@ def _eval_result_from_sdk_result( if metrics: score = _mean(list(metrics.values())) else: - score = _mean([ - 1.0 if _status_passed(getattr(run, "final_eval_status", None)) else 0.0 - for run in run_list - ]) + score = _mean([1.0 if _status_passed(getattr(run, "final_eval_status", None)) else 0.0 for run in run_list]) score = round(score, 6) - passed = bool(run_list) and all( - _status_passed(getattr(run, "final_eval_status", None)) - for run in run_list - ) + passed = bool(run_list) and all(_status_passed(getattr(run, "final_eval_status", None)) for run in run_list) failure_reason, evidence, failure_category = _failure_details(run_list) actual_invocation = _last_actual_invocation(run_list) trace_available = actual_invocation is not None - trace_payload = ( - { - "user_content": _safe_jsonable( - getattr(actual_invocation, "user_content", None) - ), - "final_response": _safe_jsonable( - getattr(actual_invocation, "final_response", None) - ), - "intermediate_data": _safe_jsonable( - getattr(actual_invocation, "intermediate_data", None) - ), - } - if actual_invocation is not None - else {} - ) - output = ( - _content_text(getattr(actual_invocation, "final_response", None)) - if actual_invocation is not None - else "" - ) + trace_payload = ({ + "user_content": _safe_jsonable(getattr(actual_invocation, "user_content", None)), + "final_response": _safe_jsonable(getattr(actual_invocation, "final_response", None)), + "intermediate_data": _safe_jsonable(getattr(actual_invocation, "intermediate_data", None)), + } if actual_invocation is not None else {}) + output = (_content_text(getattr(actual_invocation, "final_response", None)) + if actual_invocation is not None else "") case_results.append( CaseResult( case_id=case_id, @@ -1139,14 +1069,9 @@ def _eval_result_from_sdk_result( cost=0.0, hard_failed=(not passed and score <= 0.0), expected_failure_category=expected_case.expected_failure_category, - ) - ) + )) - aggregate_score = ( - round(_mean([case.score for case in case_results]), 6) - if case_results - else 0.0 - ) + aggregate_score = (round(_mean([case.score for case in case_results]), 6) if case_results else 0.0) return EvalResult( prompt_id=prompt_id, split=split, @@ -1162,9 +1087,7 @@ def _optional_num_runs(set_result: Any, *, eval_set_id: str) -> int | None: return None num_runs = getattr(set_result, "num_runs") if isinstance(num_runs, bool) or not isinstance(num_runs, int) or num_runs <= 0: - raise ValueError( - f"SDK eval set {eval_set_id!r} num_runs must be a positive integer" - ) + raise ValueError(f"SDK eval set {eval_set_id!r} num_runs must be a positive integer") return num_runs @@ -1179,31 +1102,20 @@ def _validate_sdk_run_ids( for run in runs: internal_eval_id = getattr(run, "eval_id", None) if internal_eval_id not in (None, "") and str(internal_eval_id) != eval_id: - raise ValueError( - f"SDK run internal eval_id {internal_eval_id!r} does not match " - f"container case id {eval_id!r}" - ) + raise ValueError(f"SDK run internal eval_id {internal_eval_id!r} does not match " + f"container case id {eval_id!r}") internal_eval_set_id = getattr(run, "eval_set_id", None) - if ( - internal_eval_set_id not in (None, "") - and str(internal_eval_set_id) != eval_set_id - ): - raise ValueError( - f"SDK run internal eval_set_id {internal_eval_set_id!r} does not match " - f"container eval set id {eval_set_id!r}" - ) + if (internal_eval_set_id not in (None, "") and str(internal_eval_set_id) != eval_set_id): + raise ValueError(f"SDK run internal eval_set_id {internal_eval_set_id!r} does not match " + f"container eval set id {eval_set_id!r}") run_id = getattr(run, "run_id", None) if run_id is None: continue if isinstance(run_id, bool) or not isinstance(run_id, int) or run_id <= 0: - raise ValueError( - f"SDK case {eval_id!r} run_id must be a positive integer or None" - ) + raise ValueError(f"SDK case {eval_id!r} run_id must be a positive integer or None") if num_runs is not None and run_id > num_runs: - raise ValueError( - f"SDK case {eval_id!r} run_id {run_id} exceeds num_runs={num_runs}" - ) + raise ValueError(f"SDK case {eval_id!r} run_id {run_id} exceeds num_runs={num_runs}") if run_id in seen_run_ids: raise ValueError(f"SDK evaluation result contains duplicate run_id {run_id} for case {eval_id}") seen_run_ids.add(run_id) @@ -1224,10 +1136,7 @@ def _aggregate_case_metrics(runs: list[Any], *, case_id: str) -> dict[str, float context=f"SDK case {case_id} metric {metric_name} score", ) scores_by_metric.setdefault(metric_name, []).append(score) - return { - metric_name: round(_mean(scores), 6) - for metric_name, scores in scores_by_metric.items() - } + return {metric_name: round(_mean(scores), 6) for metric_name, scores in scores_by_metric.items()} def _mean(values: list[float]) -> float: @@ -1335,6 +1244,5 @@ def _render_prompt_bundle_diff( candidate_prompts.get(name, ""), before_name=f"baseline/{name}.txt", after_name=f"{candidate_id}/{name}.txt", - ) - ) + )) return "\n\n".join(diffs) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/config.py b/examples/optimization/eval_optimize_loop/eval_loop/config.py index 5e80865f..4af66972 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/config.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/config.py @@ -114,10 +114,8 @@ def resolve_effective_seed( ) if top_present and isinstance(top_seed, int) and not isinstance(top_seed, bool): if top_seed != nested: - raise ValueError( - f"{path_text}: conflicting seed values: top-level seed={top_seed}, " - f"optimize.algorithm.seed={nested}" - ) + raise ValueError(f"{path_text}: conflicting seed values: top-level seed={top_seed}, " + f"optimize.algorithm.seed={nested}") return nested if not top_present: @@ -143,7 +141,10 @@ def validate_inputs( config: OptimizerConfig, ) -> None: validate_distinct_file_paths( - {"train": train_path, "validation": val_path}, + { + "train": train_path, + "validation": val_path + }, context="train and validation evalset paths", ) @@ -155,16 +156,11 @@ def validate_inputs( raise ValueError(f"{val_path}: validation evalset must contain at least 3 cases") validation_ids = {case.case_id for case in validation_cases} - missing_protected = [ - case_id - for case_id in config.gate.protected_case_ids - if case_id not in validation_ids - ] + missing_protected = [case_id for case_id in config.gate.protected_case_ids if case_id not in validation_ids] if missing_protected: raise ValueError( f"{optimizer_config_path}: field 'gate.protected_case_ids' references missing validation cases: " - f"{missing_protected}" - ) + f"{missing_protected}") def _parse_gate_config(payload: dict[str, Any], *, path: str) -> GateConfig: @@ -199,15 +195,11 @@ def _parse_gate_config(payload: dict[str, Any], *, path: str) -> GateConfig: ) max_total_cost_value = payload.get("max_total_cost", 1.0) - max_total_cost = ( - None - if max_total_cost_value is None - else _finite_number( - max_total_cost_value, - f"{path}: field 'gate.max_total_cost'", - 0.0, - ) - ) + max_total_cost = (None if max_total_cost_value is None else _finite_number( + max_total_cost_value, + f"{path}: field 'gate.max_total_cost'", + 0.0, + )) return GateConfig( min_val_score_improvement=min_val, @@ -236,9 +228,7 @@ def _finite_number( if number < minimum: raise ValueError(f"{field_name} must be a finite number greater than or equal to {minimum:g}") if maximum is not None and number > maximum: - raise ValueError( - f"{field_name} must be a finite number between {minimum:g} and {maximum:g}" - ) + raise ValueError(f"{field_name} must be a finite number between {minimum:g} and {maximum:g}") return number diff --git a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py index 89e0b2e5..3019a998 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py @@ -62,8 +62,7 @@ def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], sp cost=cost, hard_failed=(not judged.passed and judged.score <= 0.0), expected_failure_category=case.expected_failure_category, - ) - ) + )) score = round(sum(case.score for case in case_results) / len(case_results), 6) if case_results else 0.0 total_cost = round(sum(case.cost for case in case_results), 6) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py index 678023de..73386fa2 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py @@ -50,7 +50,10 @@ def _score_json(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="json_parse_failure", evidence=f"json parser failed at char {exc.pos}: {exc.msg}", - trace={"expectation_type": "json", "valid_json": False}, + trace={ + "expectation_type": "json", + "valid_json": False + }, ) if not isinstance(parsed, dict): return JudgeOutcome( @@ -58,7 +61,11 @@ def _score_json(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="json_value_mismatch", evidence=f"expected JSON object, got {type(parsed).__name__}", - trace={"expectation_type": "json", "valid_json": True, "object": False}, + trace={ + "expectation_type": "json", + "valid_json": True, + "object": False + }, ) required_keys = list(case.expectation.get("required_keys") or []) for key in required_keys: @@ -68,7 +75,11 @@ def _score_json(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="required_key_missing", evidence=f"missing key {key!r}; got keys {sorted(parsed.keys())!r}", - trace={"expectation_type": "json", "valid_json": True, "missing_key": key}, + trace={ + "expectation_type": "json", + "valid_json": True, + "missing_key": key + }, ) expected_values = dict(case.expectation.get("expected_values") or {}) for key, expected_value in expected_values.items(): @@ -79,7 +90,11 @@ def _score_json(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="json_value_mismatch", evidence=f"{key!r}: expected {expected_value!r}, got {actual_value!r}", - trace={"expectation_type": "json", "valid_json": True, "mismatch_key": key}, + trace={ + "expectation_type": "json", + "valid_json": True, + "mismatch_key": key + }, ) return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "json", "valid_json": True}) @@ -91,7 +106,10 @@ def _score_exact(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="exact_answer_mismatch", evidence=f"expected normalized {expected!r}, got {output!r}", - trace={"expectation_type": "exact", "expected": expected}, + trace={ + "expectation_type": "exact", + "expected": expected + }, ) return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "exact", "expected": expected}) @@ -105,7 +123,10 @@ def _score_rubric(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="forbidden_pattern", evidence=f"forbidden pattern {pattern!r} was present", - trace={"expectation_type": "rubric", "forbidden_pattern": pattern}, + trace={ + "expectation_type": "rubric", + "forbidden_pattern": pattern + }, ) must_include = [str(item) for item in case.expectation.get("must_include") or []] @@ -116,7 +137,10 @@ def _score_rubric(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="missing_rubric_terms", evidence=f"missing terms: {missing!r}", - trace={"expectation_type": "rubric", "missing_terms": missing}, + trace={ + "expectation_type": "rubric", + "missing_terms": missing + }, ) max_chars = case.expectation.get("max_chars") @@ -126,7 +150,11 @@ def _score_rubric(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="max_chars_exceeded", evidence=f"length {len(output)} exceeded max_chars {max_chars}", - trace={"expectation_type": "rubric", "length": len(output), "max_chars": int(max_chars)}, + trace={ + "expectation_type": "rubric", + "length": len(output), + "max_chars": int(max_chars) + }, ) return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "rubric"}) @@ -140,7 +168,10 @@ def _score_tool(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="tool_call_error", evidence=f"tool output was not JSON at char {exc.pos}: {exc.msg}", - trace={"expectation_type": "tool", "valid_json": False}, + trace={ + "expectation_type": "tool", + "valid_json": False + }, ) expected_tool = case.expectation.get("expected_tool") if parsed.get("tool") != expected_tool: @@ -149,7 +180,10 @@ def _score_tool(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="tool_call_error", evidence=f"expected tool {expected_tool!r}, got {parsed.get('tool')!r}", - trace={"expectation_type": "tool", "expected_tool": expected_tool}, + trace={ + "expectation_type": "tool", + "expected_tool": expected_tool + }, ) expected_args = dict(case.expectation.get("expected_args") or {}) actual_args = parsed.get("args") or {} @@ -160,7 +194,10 @@ def _score_tool(self, case: EvalCase, output: str) -> JudgeOutcome: passed=False, error_code="parameter_error", evidence=f"arg {key!r}: expected {expected_value!r}, got {actual_args.get(key)!r}", - trace={"expectation_type": "tool", "arg": key}, + trace={ + "expectation_type": "tool", + "arg": key + }, ) return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "tool"}) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py index f23d56ce..0588648d 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import Any -_ASSIGNMENT_PATTERN = re.compile(r"(? 0 and val_delta <= 0 if overfit_detected: - reasons.append( - "reject: overfit detected because train score improved but " - "validation score regressed or did not improve " - f"({train_delta:+.3f} train, {val_delta:+.3f} validation)" - ) + reasons.append("reject: overfit detected because train score improved but " + "validation score regressed or did not improve " + f"({train_delta:+.3f} train, {val_delta:+.3f} validation)") min_val_improvement = float(self.config["min_val_score_improvement"]) if val_delta < min_val_improvement: - reasons.append( - "reject: validation improvement " - f"{val_delta:+.3f} is below required {min_val_improvement:+.3f}" - ) + reasons.append("reject: validation improvement " + f"{val_delta:+.3f} is below required {min_val_improvement:+.3f}") validation_new_failures: list[str] = [] if validation_comparable: baseline_validation_by_id = baseline_validation.by_case_id() candidate_validation_by_id = candidate_validation.by_case_id() validation_new_failures = [ - case_id - for case_id, candidate_case in sorted(candidate_validation_by_id.items()) + case_id for case_id, candidate_case in sorted(candidate_validation_by_id.items()) if not candidate_case.passed and baseline_validation_by_id[case_id].passed ] if validation_new_failures: @@ -92,30 +86,23 @@ def decide( if not comparable: continue baseline_by_id = baseline_result.by_case_id() - new_hard_failures.extend( - case_id - for case_id, candidate_case in sorted(candidate_result.by_case_id().items()) - if candidate_case.hard_failed and not baseline_by_id[case_id].hard_failed - ) + new_hard_failures.extend(case_id + for case_id, candidate_case in sorted(candidate_result.by_case_id().items()) + if candidate_case.hard_failed and not baseline_by_id[case_id].hard_failed) new_hard_failures = sorted(set(new_hard_failures)) if new_hard_failures and not bool(self.config["allow_new_hard_fail"]): reasons.append(f"reject: new hard failures appeared: {new_hard_failures}") protected_ids = set(str(item) for item in self.config["protected_case_ids"]) protected_regressions = [ - delta.case_id - for delta in deltas + delta.case_id for delta in deltas if delta.split == "validation" and delta.case_id in protected_ids and delta.delta < 0 ] if protected_regressions: reasons.append(f"reject: protected cases regressed: {protected_regressions}") max_drop = float(self.config["max_score_drop_per_case"]) - excessive_drops = [ - delta.case_id - for delta in deltas - if delta.split == "validation" and delta.delta < -max_drop - ] + excessive_drops = [delta.case_id for delta in deltas if delta.split == "validation" and delta.delta < -max_drop] if excessive_drops: reasons.append(f"reject: per-case validation score drops exceed {max_drop:.3f}: {excessive_drops}") @@ -126,16 +113,12 @@ def decide( if not cost_summary.complete: reasons.append("reject: cost_unavailable for configured max_total_cost") elif total_run_cost > max_total_cost: - reasons.append( - f"reject: total run cost {total_run_cost:.3f} exceeds budget {max_total_cost:.3f}" - ) + reasons.append(f"reject: total run cost {total_run_cost:.3f} exceeds budget {max_total_cost:.3f}") accepted = not any(reason.startswith("reject:") for reason in reasons) if accepted: - reasons.append( - "accept: validation score improved " - f"{val_delta:+.3f} with no protected regression or new hard failure" - ) + reasons.append("accept: validation score improved " + f"{val_delta:+.3f} with no protected regression or new hard failure") return GateDecision( candidate_id=candidate_id, @@ -180,24 +163,18 @@ def _append_comparability_reasons( baseline_duplicates = _duplicates(baseline_ids) candidate_duplicates = _duplicates(candidate_ids) if baseline_duplicates: - reasons.append( - f"reject: baseline {split} has duplicate case IDs: {baseline_duplicates}" - ) + reasons.append(f"reject: baseline {split} has duplicate case IDs: {baseline_duplicates}") comparable = False if candidate_duplicates: - reasons.append( - f"reject: candidate {split} has duplicate case IDs: {candidate_duplicates}" - ) + reasons.append(f"reject: candidate {split} has duplicate case IDs: {candidate_duplicates}") comparable = False baseline_set = set(baseline_ids) candidate_set = set(candidate_ids) if baseline_set != candidate_set: - reasons.append( - f"reject: {split} case ID set mismatch; " - f"missing={sorted(baseline_set - candidate_set)}, " - f"extra={sorted(candidate_set - baseline_set)}" - ) + reasons.append(f"reject: {split} case ID set mismatch; " + f"missing={sorted(baseline_set - candidate_set)}, " + f"extra={sorted(candidate_set - baseline_set)}") comparable = False return comparable diff --git a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py index 64f32947..0ac1559b 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py @@ -29,23 +29,15 @@ def propose( raise TypeError("failure_summary must be a dict") if baseline_train.split != "train": - raise ValueError( - "baseline_train.split must be 'train'; " - f"got {baseline_train.split!r}" - ) + raise ValueError("baseline_train.split must be 'train'; " + f"got {baseline_train.split!r}") for case in baseline_train.cases: if case.split != "train": - raise ValueError( - f"baseline_train case {case.case_id!r} split must be 'train'; " - f"got {case.split!r}" - ) + raise ValueError(f"baseline_train case {case.case_id!r} split must be 'train'; " + f"got {case.split!r}") failed_cases = [case for case in baseline_train.cases if not case.passed] - observed_counts = Counter( - case.failure_category - for case in failed_cases - if case.failure_category - ) + observed_counts = Counter(case.failure_category for case in failed_cases if case.failure_category) _validate_failure_summary(failure_summary, observed_counts) targeted = sorted(observed_counts.keys() & _TARGET_FAILURE_CATEGORIES) @@ -59,10 +51,8 @@ def propose( CandidatePrompt( candidate_id="candidate_001_overfit", prompt=overfit_prompt, - rationale=( - f"Observed training failures ({evidence}); this candidate deliberately " - "tests the risky global-JSON correction." - ), + rationale=(f"Observed training failures ({evidence}); this candidate deliberately " + "tests the risky global-JSON correction."), prompt_diff=make_unified_diff( baseline_prompt, overfit_prompt, @@ -74,10 +64,8 @@ def propose( CandidatePrompt( candidate_id="candidate_002_safe", prompt=safe_prompt, - rationale=( - f"Observed training failures ({evidence}); this candidate limits strict " - "JSON behavior to explicit user requests." - ), + rationale=(f"Observed training failures ({evidence}); this candidate limits strict " + "JSON behavior to explicit user requests."), prompt_diff=make_unified_diff( baseline_prompt, safe_prompt, @@ -105,18 +93,14 @@ def _validate_failure_summary( summary_counts[category] = _normalize_positive_count(category, count) if summary_counts != dict(observed_counts): - raise ValueError( - "failure_summary['by_category'] must match failed train cases exactly; " - f"summary={summary_counts!r}, observed={dict(observed_counts)!r}" - ) + raise ValueError("failure_summary['by_category'] must match failed train cases exactly; " + f"summary={summary_counts!r}, observed={dict(observed_counts)!r}") def _normalize_positive_count(category: object, value: object) -> int: normalized = value if type(value) is int and value > 0 else None if normalized is None: - raise ValueError( - "failure_summary['by_category'] count must be a positive integer; " - f"category={category!r}, count={value!r}" - ) + raise ValueError("failure_summary['by_category'] count must be a positive integer; " + f"category={category!r}, count={value!r}") return normalized diff --git a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py index f605cbed..bbded42c 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py @@ -119,7 +119,10 @@ async def execute_pipeline( raise ValueError("target_prompt_paths must not be empty") _validate_target_prompt_fields(request.target_prompt_paths) validate_distinct_file_paths( - {"train": request.train_path, "validation": request.validation_path}, + { + "train": request.train_path, + "validation": request.validation_path + }, context="train and validation evalset paths", ) artifact_paths = reserve_run_artifacts(request.output_dir, run_id=run_id) @@ -394,21 +397,16 @@ async def _execute_with_snapshots( train_result = record["train_result"] validation_result = record["validation_result"] - comparable = _result_pair_is_comparable(baseline_train, train_result) and ( - _result_pair_is_comparable(baseline_validation, validation_result) - ) + comparable = _result_pair_is_comparable(baseline_train, train_result) and (_result_pair_is_comparable( + baseline_validation, validation_result)) all_results_comparable = all_results_comparable and comparable - deltas = ( - compute_case_deltas( - candidate_id=candidate.candidate_id, - baseline_train=baseline_train, - baseline_validation=baseline_validation, - candidate_train=train_result, - candidate_validation=validation_result, - ) - if comparable - else [] - ) + deltas = (compute_case_deltas( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + ) if comparable else []) decision = gate.decide( candidate_id=candidate.candidate_id, baseline_train=baseline_train, @@ -443,9 +441,8 @@ async def _execute_with_snapshots( selected_candidate=selected_candidate, update_source=request.update_source, before_hashes=before_hashes, - candidate_hashes=( - _prompt_bundle_hashes(candidate_bundles[selected_candidate]) if selected_candidate is not None else {} - ), + candidate_hashes=(_prompt_bundle_hashes(candidate_bundles[selected_candidate]) + if selected_candidate is not None else {}), input_hashes=input_snapshots.hashes(), ) audit = _build_audit( @@ -474,11 +471,8 @@ async def _execute_with_snapshots( provisional_writeback = WritebackResult( status="not_requested", before_hashes=before_hashes, - error=( - "pending source writeback; pre-write audit must be prepared before commit" - if request.update_source - else None - ), + error=("pending source writeback; pre-write audit must be prepared before commit" + if request.update_source else None), ) report = build_report( run=run, @@ -610,16 +604,17 @@ async def _execute_with_snapshots( persist_writeback_journal(artifact_paths, final_journal) except Exception as persist_error: unknown_journal = dict(final_journal) - unknown_journal.update( - { - "state": "unknown", - "error": _sanitize_error_message( - f"failed to persist terminal writeback outcome: {persist_error}", - request, - ), - "observed_hashes": _current_prompt_hashes(prompt_snapshot), - } - ) + unknown_journal.update({ + "state": + "unknown", + "error": + _sanitize_error_message( + f"failed to persist terminal writeback outcome: {persist_error}", + request, + ), + "observed_hashes": + _current_prompt_hashes(prompt_snapshot), + }) try: persist_writeback_journal(artifact_paths, unknown_journal) except Exception: @@ -749,14 +744,8 @@ def _validate_backend_seed(backend: Any, effective_seed: int) -> None: if not hasattr(backend, "seed"): return backend_seed = getattr(backend, "seed") - if ( - isinstance(backend_seed, bool) - or not isinstance(backend_seed, int) - or backend_seed != effective_seed - ): - raise ValueError( - f"fake backend seed {backend_seed!r} does not match effective seed {effective_seed}" - ) + if (isinstance(backend_seed, bool) or not isinstance(backend_seed, int) or backend_seed != effective_seed): + raise ValueError(f"fake backend seed {backend_seed!r} does not match effective seed {effective_seed}") def _strict_case_metadata(path: str | Path, *, role: str) -> dict[str, bool]: @@ -764,9 +753,7 @@ def _strict_case_metadata(path: str | Path, *, role: str) -> dict[str, bool]: has_standard = "eval_cases" in payload has_legacy = "cases" in payload if has_standard == has_legacy: - raise ValueError( - f"{role} evalset must contain exactly one of eval_cases or cases" - ) + raise ValueError(f"{role} evalset must contain exactly one of eval_cases or cases") if has_standard: cases = payload["eval_cases"] standard = True @@ -869,18 +856,10 @@ def _verify_prompt_integrity( try: observed[name] = hashlib.sha256(prompt_file.path.read_bytes()).hexdigest() except OSError as error: - raise PromptRestorationError( - f"source prompt field {name!r} is unreadable {context}" - ) from error + raise PromptRestorationError(f"source prompt field {name!r} is unreadable {context}") from error if observed != expected: - changed = sorted( - name - for name in set(expected) | set(observed) - if expected.get(name) != observed.get(name) - ) - raise ConcurrentPromptUpdateError( - f"source prompt integrity changed {context}: {', '.join(changed)}" - ) + changed = sorted(name for name in set(expected) | set(observed) if expected.get(name) != observed.get(name)) + raise ConcurrentPromptUpdateError(f"source prompt integrity changed {context}: {', '.join(changed)}") async def _await_backend_with_prompt_integrity( @@ -930,14 +909,12 @@ def _verify_terminal_writeback_integrity( ) except (ConcurrentPromptUpdateError, PromptRestorationError) as integrity_error: unknown_journal = dict(journal) - unknown_journal.update( - { - "state": "unknown", - "report_phase": "final", - "after_hashes": _current_prompt_hashes(snapshot), - "error": _sanitize_error_message(str(integrity_error), request), - } - ) + unknown_journal.update({ + "state": "unknown", + "report_phase": "final", + "after_hashes": _current_prompt_hashes(snapshot), + "error": _sanitize_error_message(str(integrity_error), request), + }) try: persist_writeback_journal(artifact_paths, unknown_journal) except Exception as persist_error: @@ -955,14 +932,12 @@ def _terminal_journal( input_drift: dict[str, Any] | None = None, ) -> dict[str, Any]: terminal = dict(journal) - terminal.update( - { - "state": state, - "report_phase": "final", - "after_hashes": dict(writeback.after_hashes), - "error": writeback.error, - } - ) + terminal.update({ + "state": state, + "report_phase": "final", + "after_hashes": dict(writeback.after_hashes), + "error": writeback.error, + }) if input_drift: terminal["input_drift"] = input_drift return terminal @@ -1041,10 +1016,7 @@ def _recoverable_candidate_evaluation_failure( """ _verify_snapshot_integrity(input_snapshots) - if _exception_chain_contains( - error, - (ConcurrentPromptUpdateError, PromptRestorationError), - ): + if _exception_chain_contains(error, (ConcurrentPromptUpdateError, PromptRestorationError)): raise error try: _verify_prompt_integrity( @@ -1059,9 +1031,7 @@ def _recoverable_candidate_evaluation_failure( "stage": stage, "type": type(error).__name__, "message": "candidate evaluation failed; backend details withheld", - "message_sha256": hashlib.sha256( - raw_message.encode("utf-8", errors="replace") - ).hexdigest(), + "message_sha256": hashlib.sha256(raw_message.encode("utf-8", errors="replace")).hexdigest(), "completed_splits": [result.split for result in completed_results], "known_evaluator_cost": round(sum(result.cost for result in completed_results), 6), "cost_complete": False, @@ -1097,10 +1067,8 @@ def _evaluation_error_gate_decision( candidate_cost = round(float(failure["known_evaluator_cost"]), 6) total_run_cost = round(cumulative_cost + candidate_cost, 6) stage = str(failure["stage"]) - reason = ( - f"reject: evaluation_error during {stage}: " - f"{failure['type']}: {failure['message']}" - ) + reason = (f"reject: evaluation_error during {stage}: " + f"{failure['type']}: {failure['message']}") return GateDecision( candidate_id=candidate_id, accepted=False, @@ -1159,16 +1127,13 @@ def _validated_candidate_bundles( bundle = candidate.bundle() actual_fields = set(bundle) if actual_fields != expected_fields: - raise ValueError( - f"candidate {candidate_id!r} bundle fields must exactly match target prompt fields; " - f"missing={sorted(expected_fields - actual_fields)}, " - f"extra={sorted(actual_fields - expected_fields)}" - ) + raise ValueError(f"candidate {candidate_id!r} bundle fields must exactly match target prompt fields; " + f"missing={sorted(expected_fields - actual_fields)}, " + f"extra={sorted(actual_fields - expected_fields)}") for field_name, prompt_text in bundle.items(): if not isinstance(prompt_text, str) or not prompt_text: - raise ValueError( - f"candidate {candidate_id!r} bundle field {field_name!r} " "must be a non-empty string" - ) + raise ValueError(f"candidate {candidate_id!r} bundle field {field_name!r} " + "must be a non-empty string") _validate_utf8_text( prompt_text, context=f"candidate {candidate_id!r} bundle field {field_name!r}", @@ -1196,15 +1161,9 @@ def _validate_utf8_text(value: Any, *, context: str) -> None: def _result_pair_is_comparable(baseline: EvalResult, candidate: EvalResult) -> bool: baseline_ids = [case.case_id for case in baseline.cases] candidate_ids = [case.case_id for case in candidate.cases] - return ( - bool(baseline_ids) - and bool(candidate_ids) - and ( - len(baseline_ids) == len(set(baseline_ids)) - and len(candidate_ids) == len(set(candidate_ids)) - and set(baseline_ids) == set(candidate_ids) - ) - ) + return (bool(baseline_ids) and bool(candidate_ids) + and (len(baseline_ids) == len(set(baseline_ids)) and len(candidate_ids) == len(set(candidate_ids)) + and set(baseline_ids) == set(candidate_ids))) def _validate_eval_result( @@ -1215,14 +1174,11 @@ def _validate_eval_result( expected_case_ids: set[str] | None = None, ) -> None: if result.prompt_id != expected_prompt_id: - raise ValueError( - f"evaluation result prompt_id mismatch: expected {expected_prompt_id!r}, " f"got {result.prompt_id!r}" - ) + raise ValueError(f"evaluation result prompt_id mismatch: expected {expected_prompt_id!r}, " + f"got {result.prompt_id!r}") if result.split != expected_split: - raise ValueError( - f"EvalResult split mismatch for {expected_prompt_id!r}: " - f"expected {expected_split!r}, got {result.split!r}" - ) + raise ValueError(f"EvalResult split mismatch for {expected_prompt_id!r}: " + f"expected {expected_split!r}, got {result.split!r}") score = _finite_number( result.score, context=f"EvalResult score for {expected_prompt_id!r}", @@ -1246,10 +1202,8 @@ def _validate_eval_result( pass case_ids.add(case.case_id) if case.split != expected_split: - raise ValueError( - f"CaseResult {case.case_id!r} split mismatch for {expected_prompt_id!r}: " - f"expected {expected_split!r}, got {case.split!r}" - ) + raise ValueError(f"CaseResult {case.case_id!r} split mismatch for {expected_prompt_id!r}: " + f"expected {expected_split!r}, got {case.split!r}") _finite_number( case.score, context=f"CaseResult {case.case_id!r} score", @@ -1282,10 +1236,8 @@ def _validate_eval_result( if expected_case_ids is not None: observed_ids = [case.case_id for case in result.cases] if len(observed_ids) != len(set(observed_ids)) or set(observed_ids) != expected_case_ids: - raise ValueError( - f"{expected_prompt_id} {expected_split} result must exactly match dataset case IDs; " - f"expected={sorted(expected_case_ids)}, observed={sorted(set(observed_ids))}" - ) + raise ValueError(f"{expected_prompt_id} {expected_split} result must exactly match dataset case IDs; " + f"expected={sorted(expected_case_ids)}, observed={sorted(set(observed_ids))}") def _validate_optimization_result(result: OptimizationResult) -> None: @@ -1293,11 +1245,8 @@ def _validate_optimization_result(result: OptimizationResult) -> None: raise ValueError("optimization result candidates and rounds must be lists") _validate_cost_summary(result.cost, context="optimization cost") for index, round_record in enumerate(result.rounds): - if ( - isinstance(round_record.round_id, bool) - or not isinstance(round_record.round_id, int) - or round_record.round_id <= 0 - ): + if (isinstance(round_record.round_id, bool) or not isinstance(round_record.round_id, int) + or round_record.round_id <= 0): raise ValueError(f"optimization round {index} round_id must be a positive integer") _validate_candidate_id(round_record.candidate_id) _finite_number( @@ -1338,10 +1287,10 @@ def _validate_cost_summary(summary: CostSummary, *, context: str) -> None: minimum=0.0, ) if summary.complete and not math.isclose( - total, - sum(components.values()), - rel_tol=0.0, - abs_tol=1e-6, + total, + sum(components.values()), + rel_tol=0.0, + abs_tol=1e-6, ): raise ValueError(f"{context} total must equal components when complete") @@ -1403,8 +1352,7 @@ def _gate_config_with_dataset_protection( effective = dict(gate_config) configured_ids = effective.get("protected_case_ids", []) if not isinstance(configured_ids, list) or not all( - isinstance(case_id, str) and bool(case_id.strip()) for case_id in configured_ids - ): + isinstance(case_id, str) and bool(case_id.strip()) for case_id in configured_ids): raise ValueError("gate protected_case_ids must be a list of non-empty strings") missing_ids = sorted(set(configured_ids) - set(validation_metadata)) if missing_ids: @@ -1424,11 +1372,7 @@ def _select_candidate( for index, record in enumerate(candidate_records): candidate = record["candidate"] decision = decisions_by_id[candidate.candidate_id] - if ( - decision.accepted - and "train_result" in record - and "validation_result" in record - ): + if (decision.accepted and "train_result" in record and "validation_result" in record): accepted.append((index, record)) if not accepted: return None @@ -1492,23 +1436,23 @@ def _build_audit( writeback_journal: dict[str, Any], ) -> dict[str, Any]: candidate_costs = { - record["candidate"].candidate_id: round( - sum( - record[result_name].cost - for result_name in ("train_result", "validation_result") - if result_name in record - ), + record["candidate"].candidate_id: + round( + sum(record[result_name].cost for result_name in ("train_result", "validation_result") + if result_name in record), 6, ) for record in candidate_records } candidate_evaluation_failures = { record["candidate"].candidate_id: dict(record["evaluation_error"]) - for record in candidate_records - if "evaluation_error" in record + for record in candidate_records if "evaluation_error" in record } candidate_prompt_hashes = { - candidate_id: {name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() for name, prompt in bundle.items()} + candidate_id: { + name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() + for name, prompt in bundle.items() + } for candidate_id, bundle in candidate_bundles.items() } cost_audit: dict[str, Any] = { @@ -1541,7 +1485,9 @@ def _build_audit( "validation": _display_path(request.validation_path), "optimizer": _display_path(request.optimizer_config_path), "prompts": target_paths, - **({"prompt": target_paths["system_prompt"]} if "system_prompt" in target_paths else {}), + **({ + "prompt": target_paths["system_prompt"] + } if "system_prompt" in target_paths else {}), }, "prompt_hash": snapshot_hashes.get("system_prompt"), "prompt_hashes": snapshot_hashes, @@ -1554,11 +1500,13 @@ def _build_audit( for index, record in enumerate(candidate_records, start=1) }, "candidate_prompts": { - candidate_id: dict(bundle) for candidate_id, bundle in candidate_bundles.items() + candidate_id: dict(bundle) + for candidate_id, bundle in candidate_bundles.items() }, "candidate_evaluation_failures": candidate_evaluation_failures, "prompt_diffs": { - record["candidate"].candidate_id: record["candidate"].prompt_diff for record in candidate_records + record["candidate"].candidate_id: record["candidate"].prompt_diff + for record in candidate_records }, "total_run_cost": cost_summary.total if cost_summary.complete else None, "known_run_cost": cost_summary.total if not cost_summary.complete else None, @@ -1566,12 +1514,12 @@ def _build_audit( "cost": cost_audit, "sdk_result_summary": _sanitize_public_value(to_jsonable(optimization_raw_summary)), "sdk_result_availability": { - "aggregate_validation_result": bool(request.mode == "sdk" and optimization_raw_summary), - "full_train_eval_result": not any( - failure.get("stage") == "train" - for failure in candidate_evaluation_failures.values() - ), - "full_per_case_validation_delta": all_results_comparable, + "aggregate_validation_result": + bool(request.mode == "sdk" and optimization_raw_summary), + "full_train_eval_result": + not any(failure.get("stage") == "train" for failure in candidate_evaluation_failures.values()), + "full_per_case_validation_delta": + all_results_comparable, }, "reproducibility_shell": "powershell", "reproducibility_command": _reproducibility_command(request), @@ -1634,37 +1582,34 @@ def _pretty_json_sha256(value: Any) -> str: def _is_secret_key(lowered: str) -> bool: normalized = lowered.replace("-", "_").replace(" ", "_") if normalized in { - "api_key", - "apikey", - "token", - "access_token", - "refresh_token", - "auth_token", - "password", - "passwd", - "credentials", - "credential", - "secret", - "authorization", - "private_key", - "signing_key", - "ssh_key", - }: - return True - if normalized.endswith("_token"): - return True - return any( - marker in normalized - for marker in ( "api_key", + "apikey", + "token", + "access_token", + "refresh_token", + "auth_token", "password", "passwd", + "credentials", "credential", "secret", "authorization", "private_key", - ) - ) + "signing_key", + "ssh_key", + }: + return True + if normalized.endswith("_token"): + return True + return any(marker in normalized for marker in ( + "api_key", + "password", + "passwd", + "credential", + "secret", + "authorization", + "private_key", + )) def _sanitize_public_value(value: Any) -> Any: @@ -1724,19 +1669,13 @@ def _powershell_quote(value: str) -> str: def _powershell_arg(value: str) -> str: placeholder = re.search(r"\$(OUTPUT_DIR|EXTERNAL)", value) if placeholder is not None: - prefix = value[: placeholder.start()] - suffix = value[placeholder.end() :] + prefix = value[:placeholder.start()] + suffix = value[placeholder.end():] def escape_literal(text: str) -> str: return text.replace("`", "``").replace("$", "`$").replace('"', '`"') - return ( - '"' - + escape_literal(prefix) - + placeholder.group(0) - + escape_literal(suffix) - + '"' - ) + return ('"' + escape_literal(prefix) + placeholder.group(0) + escape_literal(suffix) + '"') if re.fullmatch(r"[A-Za-z0-9_./:-]+", value): return value return _powershell_quote(value) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py index 7cc63734..1734c851 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/report.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -27,16 +27,13 @@ from .schemas import WritebackResult from .schemas import to_jsonable - -REPRODUCIBILITY_COMMAND = ( - "python examples/optimization/eval_optimize_loop/run_pipeline.py " - "--train examples/optimization/eval_optimize_loop/data/train.evalset.json " - "--val examples/optimization/eval_optimize_loop/data/val.evalset.json " - "--optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json " - "--prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt " - "--output-dir /tmp/eval-optimize-loop " - "--fake-model --fake-judge --trace" -) +REPRODUCIBILITY_COMMAND = ("python examples/optimization/eval_optimize_loop/run_pipeline.py " + "--train examples/optimization/eval_optimize_loop/data/train.evalset.json " + "--val examples/optimization/eval_optimize_loop/data/val.evalset.json " + "--optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json " + "--prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt " + "--output-dir /tmp/eval-optimize-loop " + "--fake-model --fake-judge --trace") @dataclass(frozen=True) @@ -125,8 +122,7 @@ def compute_case_deltas( candidate_passed=candidate_case.passed, regression=delta < 0, delta_type=delta_type, - ) - ) + )) return deltas @@ -154,7 +150,10 @@ def build_report( return OptimizationReport( schema_version="eval_optimize_loop.v2", run=run, - baseline={"train": baseline_train, "validation": baseline_validation}, + baseline={ + "train": baseline_train, + "validation": baseline_validation + }, baseline_train=baseline_train, baseline_validation=baseline_validation, candidates=candidate_records, @@ -210,15 +209,11 @@ def reserve_run_artifacts( final_run_dir = runs_root / safe_run_id for occupied in (final_run_dir, temp_run_dir): if os.path.lexists(occupied): - raise FileExistsError( - f"run ID {safe_run_id!r} is already reserved or published in {runs_root}" - ) + raise FileExistsError(f"run ID {safe_run_id!r} is already reserved or published in {runs_root}") try: temp_run_dir.mkdir() except FileExistsError as error: - raise FileExistsError( - f"run ID {safe_run_id!r} is already reserved or published in {runs_root}" - ) from error + raise FileExistsError(f"run ID {safe_run_id!r} is already reserved or published in {runs_root}") from error # A concurrently published final must win. Leave no misleading reservation # when our just-created directory is still empty. @@ -227,9 +222,7 @@ def reserve_run_artifacts( temp_run_dir.rmdir() except OSError: pass - raise FileExistsError( - f"run ID {safe_run_id!r} is already reserved or published in {runs_root}" - ) + raise FileExistsError(f"run ID {safe_run_id!r} is already reserved or published in {runs_root}") return RunArtifactPaths( output_dir=output_path, run_id=safe_run_id, @@ -313,10 +306,8 @@ def finalize_run_artifacts( except BaseException as manifest_error: add_note = getattr(callback_error, "add_note", None) if add_note is not None: - add_note( - "failed to refresh temp artifact manifest after before_publish failure: " - f"{manifest_error}" - ) + add_note("failed to refresh temp artifact manifest after before_publish failure: " + f"{manifest_error}") raise _durable_publish_directory(paths) @@ -356,9 +347,7 @@ def _require_mutable_temp_run(paths: RunArtifactPaths) -> None: try: metadata = os.lstat(paths.temp_run_dir) except FileNotFoundError as error: - raise FileNotFoundError( - f"reserved temp directory for run ID {paths.run_id!r} is unavailable" - ) from error + raise FileNotFoundError(f"reserved temp directory for run ID {paths.run_id!r} is unavailable") from error if paths.temp_run_dir.is_symlink() or _is_reparse_point(paths.temp_run_dir): raise OSError(f"refusing unsafe reserved temp directory: {paths.temp_run_dir}") if not stat.S_ISDIR(metadata.st_mode): @@ -371,16 +360,13 @@ def report_to_json(report: OptimizationReport) -> str: def _json_text(value: Any) -> str: - return ( - json.dumps( - to_jsonable(value), - indent=2, - ensure_ascii=False, - sort_keys=True, - allow_nan=False, - ) - + "\n" - ) + return (json.dumps( + to_jsonable(value), + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + "\n") def _atomic_write_text(path: Path, content: str) -> None: @@ -562,8 +548,7 @@ def render_markdown(report: OptimizationReport) -> str: lines.extend([ "", - "Update source prompt: " - + ("yes" if report.run.get("update_source") else "no (default)"), + "Update source prompt: " + ("yes" if report.run.get("update_source") else "no (default)"), "", ]) availability = report.audit.get("sdk_result_availability", {}) @@ -615,15 +600,9 @@ def render_markdown(report: OptimizationReport) -> str: train_result = record.get("train_result") validation_result = record.get("validation_result") train_score = f"{train_result.score:.3f}" if isinstance(train_result, EvalResult) else "n/a" - validation_score = ( - f"{validation_result.score:.3f}" - if isinstance(validation_result, EvalResult) - else "n/a" - ) - lines.append( - f"| {candidate.candidate_id} | {train_score} | " - f"{validation_score} | {verdict} |" - ) + validation_score = (f"{validation_result.score:.3f}" if isinstance(validation_result, EvalResult) else "n/a") + lines.append(f"| {candidate.candidate_id} | {train_score} | " + f"{validation_score} | {verdict} |") lines.extend([ "", @@ -633,12 +612,10 @@ def render_markdown(report: OptimizationReport) -> str: "| --- | --- | --- | ---: | ---: | ---: | --- | --- |", ]) for delta in report.per_case_deltas: - lines.append( - f"| {delta.candidate_id} | {delta.split} | {delta.case_id} | " - f"{delta.baseline_score:.3f} | {delta.candidate_score:.3f} | " - f"{delta.delta:+.3f} | {delta.baseline_passed} -> {delta.candidate_passed} | " - f"{delta.delta_type} |" - ) + lines.append(f"| {delta.candidate_id} | {delta.split} | {delta.case_id} | " + f"{delta.baseline_score:.3f} | {delta.candidate_score:.3f} | " + f"{delta.delta:+.3f} | {delta.baseline_passed} -> {delta.candidate_passed} | " + f"{delta.delta_type} |") summary = report.failure_attribution_summary lines.extend([ @@ -667,15 +644,11 @@ def render_markdown(report: OptimizationReport) -> str: else: reported_optimizer_cost = cost_audit.get("reported_optimizer_cost") if reported_optimizer_cost is not None: - lines.append( - "Reported optimizer cost (incomplete; not total run cost): " - f"{reported_optimizer_cost:.3f}" - ) + lines.append("Reported optimizer cost (incomplete; not total run cost): " + f"{reported_optimizer_cost:.3f}") lines.append(f"Known evaluator cost: {cost_audit.get('evaluator', 0):.3f}") - lines.append( - "Known run cost (incomplete; not total run cost): " - f"{cost_audit.get('known_run_cost', 0):.3f}" - ) + lines.append("Known run cost (incomplete; not total run cost): " + f"{cost_audit.get('known_run_cost', 0):.3f}") lines.append("Complete run cost: unavailable") lines.extend([ f"Config hash: `{report.audit.get('config_hash', '')}`", @@ -702,8 +675,7 @@ def render_markdown(report: OptimizationReport) -> str: "## Reproducibility", "", f"```{report.run.get('reproducibility_shell') or 'bash'}", - report.run.get("reproducibility_command") - or report.audit.get("reproducibility_command") + report.run.get("reproducibility_command") or report.audit.get("reproducibility_command") or REPRODUCIBILITY_COMMAND, "```", "", @@ -719,11 +691,11 @@ def write_audit_artifacts(report: OptimizationReport, run_dir: Path) -> None: _require_safe_audit_root(run_dir) for component in ( - "case_results", - "baseline_prompts", - "candidate_prompts", - "prompt_diffs", - "rounds", + "case_results", + "baseline_prompts", + "candidate_prompts", + "prompt_diffs", + "rounds", ): _ensure_safe_audit_directory(run_dir, component) for index, record in enumerate(report.candidates, start=1): @@ -841,11 +813,8 @@ def _write_safe_audit_text( run_root = Path(run_root) relative = Path(relative_path) - if ( - relative.is_absolute() - or not relative.parts - or any(component in {"", ".", ".."} for component in relative.parts) - ): + if (relative.is_absolute() or not relative.parts + or any(component in {"", ".", ".."} for component in relative.parts)): raise OSError(f"unsafe audit artifact path: {relative_path!r}") if len(relative.parts) == 1: _require_safe_audit_root(run_root) @@ -876,13 +845,8 @@ def _ensure_safe_audit_directory(run_root: Path, *components: str) -> Path: raise ValueError("audit directory components must not be empty") for component in components: component_path = Path(component) - if ( - not component - or component_path.is_absolute() - or len(component_path.parts) != 1 - or component_path.name != component - or component in {".", ".."} - ): + if (not component or component_path.is_absolute() or len(component_path.parts) != 1 + or component_path.name != component or component in {".", ".."}): raise OSError(f"unsafe audit directory component: {component!r}") directory = run_root.joinpath(*components) @@ -918,11 +882,9 @@ def _write_artifact_manifest( ) -> None: _require_mutable_temp_run(paths) artifacts = _manifest_artifact_layout(report) - expected_files = ( - {"writeback_journal.json": expected_journal.encode("utf-8")} - if expected_journal is not None - else {} - ) + expected_files = ({ + "writeback_journal.json": expected_journal.encode("utf-8") + } if expected_journal is not None else {}) files = _manifest_file_records(paths.temp_run_dir, expected_files=expected_files) declared_paths = _declared_manifest_paths(artifacts) if expected_journal is not None: @@ -952,11 +914,8 @@ def _manifest_artifact_layout(report: OptimizationReport) -> dict[str, Any]: artifact_id = _candidate_artifact_name(index, candidate.candidate_id) prompt_bundle = record.get("prompt_bundle") or candidate.bundle() prompt_paths = { - field_name: ( - Path("candidate_prompts") - / artifact_id - / f"{_safe_artifact_name(str(field_name))}.txt" - ).as_posix() + field_name: + (Path("candidate_prompts") / artifact_id / f"{_safe_artifact_name(str(field_name))}.txt").as_posix() for field_name in prompt_bundle } case_results: dict[str, str] = {} @@ -965,19 +924,15 @@ def _manifest_artifact_layout(report: OptimizationReport) -> dict[str, Any]: if not isinstance(result, EvalResult): continue split = _safe_artifact_name(str(result.split)) - case_results[str(result.split)] = ( - Path("case_results") / f"{artifact_id}_{split}.json" - ).as_posix() - candidates.append( - { - "candidate_id": candidate.candidate_id, - "artifact_id": artifact_id, - "prompt_bundle": prompt_paths, - "diff": (Path("prompt_diffs") / f"{artifact_id}.diff").as_posix(), - "case_results": case_results, - "evaluation_failure": "evaluation_error" in record, - } - ) + case_results[str(result.split)] = (Path("case_results") / f"{artifact_id}_{split}.json").as_posix() + candidates.append({ + "candidate_id": candidate.candidate_id, + "artifact_id": artifact_id, + "prompt_bundle": prompt_paths, + "diff": (Path("prompt_diffs") / f"{artifact_id}.diff").as_posix(), + "case_results": case_results, + "evaluation_failure": "evaluation_error" in record, + }) return { "reports": { @@ -991,15 +946,11 @@ def _manifest_artifact_layout(report: OptimizationReport) -> dict[str, Any]: "validation": "case_results/baseline_validation.json", }, "baseline_prompts": { - field_name: ( - Path("baseline_prompts") / f"{_safe_artifact_name(str(field_name))}.txt" - ).as_posix() + field_name: (Path("baseline_prompts") / f"{_safe_artifact_name(str(field_name))}.txt").as_posix() for field_name in report.baseline_prompts }, - "rounds": [ - (Path("rounds") / f"{_safe_artifact_name(str(round_record.round_id))}.json").as_posix() - for round_record in report.rounds - ], + "rounds": [(Path("rounds") / f"{_safe_artifact_name(str(round_record.round_id))}.json").as_posix() + for round_record in report.rounds], "audit_records": { "audit": "audit.json", "config_snapshot": "config.snapshot.json", @@ -1011,7 +962,8 @@ def _manifest_artifact_layout(report: OptimizationReport) -> dict[str, Any]: "writeback_journal": "writeback_journal.json", "evaluation_failures": "evaluation_failures.json", }, - "candidates": candidates, + "candidates": + candidates, } @@ -1042,14 +994,11 @@ def _manifest_file_records( if relative_path == "artifact_manifest.json": continue content_by_path[relative_path] = artifact.read_bytes() - return [ - { - "path": relative_path, - "sha256": hashlib.sha256(content).hexdigest(), - "size_bytes": len(content), - } - for relative_path, content in sorted(content_by_path.items()) - ] + return [{ + "path": relative_path, + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + } for relative_path, content in sorted(content_by_path.items())] def _candidate_artifact_name(index: int, candidate_id: str) -> str: diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py index 752b40fc..58afb147 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -30,8 +30,7 @@ def from_dict(cls, payload: dict[str, Any], split: str) -> "EvalCase": raise ValueError(f"eval case is missing id/case_id: {payload!r}") if "split" in payload and str(payload["split"]) != str(split): raise ValueError( - f"eval case {case_id!r} split mismatch: payload has {payload['split']!r}, expected {split!r}" - ) + f"eval case {case_id!r} split mismatch: payload has {payload['split']!r}, expected {split!r}") expectation = payload.get("expectation") if not isinstance(expectation, dict): raise ValueError(f"eval case {case_id!r} is missing expectation object") @@ -212,9 +211,7 @@ class OptimizationReport: audit: dict[str, Any] rounds: list[OptimizationRound] = field(default_factory=list) cost_summary: CostSummary = field(default_factory=CostSummary) - writeback: WritebackResult = field( - default_factory=lambda: WritebackResult(status="not_requested") - ) + writeback: WritebackResult = field(default_factory=lambda: WritebackResult(status="not_requested")) baseline_prompts: dict[str, str] = field(default_factory=dict) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py index d28dcb2a..f3a73468 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py @@ -206,9 +206,8 @@ def temporary_prompt_bundle( name for name, candidate_hash in candidate_hashes.items() if installed_hashes[name] != candidate_hash ] if candidate_mismatches: - raise ConcurrentPromptUpdateError( - "candidate prompt files changed before temporary evaluation: " f"{', '.join(candidate_mismatches)}" - ) + raise ConcurrentPromptUpdateError("candidate prompt files changed before temporary evaluation: " + f"{', '.join(candidate_mismatches)}") yield except BaseException as primary_error: failures = _restore_snapshot(snapshot, written_hashes) @@ -225,9 +224,7 @@ def temporary_prompt_bundle( failures = _restore_snapshot(snapshot, written_hashes) restoration_error = _restoration_error(snapshot, failures) if restoration_error is not None: - raise PromptRestorationError( - f"failed to restore prompt snapshot: {restoration_error}" - ) + raise PromptRestorationError(f"failed to restore prompt snapshot: {restoration_error}") def commit_prompt_bundle( @@ -258,9 +255,8 @@ def commit_prompt_bundle( name for name, candidate_hash in candidate_hashes.items() if applied_hashes[name] != candidate_hash ] if candidate_mismatches: - raise ConcurrentPromptUpdateError( - "candidate prompt files changed before final verification: " f"{', '.join(candidate_mismatches)}" - ) + raise ConcurrentPromptUpdateError("candidate prompt files changed before final verification: " + f"{', '.join(candidate_mismatches)}") except (OSError, ConcurrentPromptUpdateError) as error: if isinstance(error, ConcurrentPromptUpdateError) and not written_hashes: raise @@ -272,11 +268,9 @@ def commit_prompt_bundle( after_hashes = {} rollback_failures.append(f"hash verification: {hash_error}") else: - rollback_failures.extend( - f"{name}: final hash differs from snapshot" - for name, expected_hash in expected_hashes.items() - if after_hashes[name] != expected_hash - ) + rollback_failures.extend(f"{name}: final hash differs from snapshot" + for name, expected_hash in expected_hashes.items() + if after_hashes[name] != expected_hash) error_message = f"prompt commit failed: {error}" if rollback_failures: diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json index 4b41a37e..35641f69 100644 --- a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json @@ -1,51 +1,201 @@ { "audit": { + "candidate_artifacts": { + "candidate_001_overfit": "001-7cfbcadbbc09", + "candidate_002_safe": "002-ef1c1c866d24" + }, + "candidate_evaluation_failures": {}, "candidate_prompt_hashes": { - "candidate_001_overfit": "07a494c337544626a4c23afe1a945488e503b06da3e12597419470eb2ab09e42", - "candidate_002_safe": "9669bbd13891b78d2c493df624050dcfd1aa36533987f082a1b78b864df5572f" + "candidate_001_overfit": { + "system_prompt": "4b791dfd23a761db46c408b0ae0ce76549bcde4588ba256b68d6ff6083f5349f" + }, + "candidate_002_safe": { + "system_prompt": "920ffe712cf686259e940bfe712fe1cf819cbeade662639576eeca12c9464f17" + } }, "candidate_prompts": { "candidate_001_overfit": { - "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", - "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", - "rationale": "The train failures are strict JSON/exact formatting failures, so this candidate over-corrects by forcing JSON globally." + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n" }, "candidate_002_safe": { - "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\nUse strict JSON only when the user explicitly asks for JSON.\nUse exact answers only when the user explicitly asks for an exact answer.\nOtherwise answer naturally and honor rubric constraints.\n", - "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints.", - "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n" + } + }, + "config_file_sha256": "3d8c9e3f229f3ef2627dc89586952367cc456631a4b4ba20dc0313ec4098c095", + "config_hash": "7144548d6c99b0a6226534dfe51983875c26bf3193b7ad07aaecf9a1cb80dcfd", + "config_snapshot": { + "evaluate": { + "metrics": [ + { + "criterion": { + "final_response": { + "text": { + "case_insensitive": false, + "match": "exact" + } + } + }, + "metric_name": "final_response_avg_score", + "threshold": 1.0 + } + ], + "num_runs": 1 + }, + "optimize": { + "algorithm": { + "candidate_selection_strategy": "pareto", + "frontier_type": "instance", + "max_iterations_without_improvement": 3, + "max_metric_calls": 18, + "module_selector": "round_robin", + "name": "gepa_reflective", + "reflection_history_top_k": 2, + "reflection_lm": { + "api_key": "", + "base_url": "${TRPC_AGENT_BASE_URL}", + "generation_config": { + "max_tokens": 2048, + "temperature": 0.4 + }, + "model_name": "${TRPC_AGENT_MODEL_NAME}" + }, + "reflection_minibatch_size": 3, + "score_threshold": 1.0, + "seed": 91, + "skip_perfect_score": false, + "use_merge": false + }, + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + } } }, - "config_hash": "66a1db3a1c84ad12fd41385fc5e1a9c23cc305e6973945c4bade559c804f9abe", "cost": { "baseline": 0.006, "candidates": { "candidate_001_overfit": 0.006, "candidate_002_safe": 0.006 }, + "complete": true, + "evaluator": 0.018, + "optimization": { + "agent": 0.0, + "complete": true, + "evaluator": 0.0, + "optimizer": 0.0, + "reported_optimizer_cost": null, + "total": 0.0 + }, + "reported_optimizer_cost": null, "total": 0.018 }, - "duration_seconds": 0.0, + "duration_seconds": 0.031103599998459686, + "gate_config_hash": "b47dce7f4b2eeabcb7db41e70c1374d15e56c2e69876e8b82e6febdcfa1fe3ab", + "gate_config_snapshot": { + "allow_new_hard_fail": false, + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, + "min_val_score_improvement": 0.01, + "protected_case_ids": [ + "val_protected_yes_no" + ] + }, "input_hashes": { - "optimizer": "a00febaa9efaaf299bbf7dbc08eb31b9093315f0079e601d59c84d0d5704f784", - "prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95", - "train": "03ae8fd41a19ab376c733cb4d5b92a5d6c34431717b5b4899cabdb1e23351a56", - "validation": "73a32b7773f789b2da3d15e925ca0f57da05e8a19c079c9a56ef42055961804b" + "optimizer": "3d8c9e3f229f3ef2627dc89586952367cc456631a4b4ba20dc0313ec4098c095", + "target_prompts": { + "system_prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95" + }, + "train": "269e76792ff6fc2ce91c3a1008095dd020f4769d8af0b98d9f9410cc179a276e", + "validation": "292ee54dec4d33f34f3218672af3c56f4b3be2149960e19868f0de37e48e272a" }, "input_paths": { - "optimizer": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\optimizer.json", - "prompt": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", - "train": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\train.evalset.json", - "validation": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\val.evalset.json" + "optimizer": "examples/optimization/eval_optimize_loop/data/optimizer.json", + "prompt": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt", + "prompts": { + "system_prompt": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt" + }, + "train": "examples/optimization/eval_optimize_loop/data/train.evalset.json", + "validation": "examples/optimization/eval_optimize_loop/data/val.evalset.json" }, + "known_run_cost": null, "prompt_diffs": { - "candidate_001_overfit": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", - "candidate_002_safe": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints." + "candidate_001_overfit": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,5 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+Always force every final answer into JSON.", + "candidate_002_safe": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,5 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+Use strict JSON only when the user explicitly asks." }, "prompt_hash": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95", - "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", + "prompt_hashes": { + "system_prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95" + }, + "redacted_config_hash": "4eb087aac8be78ccb16e36073e6c6494cc10f13f50403f4026875ef7c113591a", + "redacted_config_snapshot_sha256": "008aa7d312d78530925264047ee5d05e8daa1147ebb78b6f225f2c72d40f8f34", + "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --output-dir \"$OUTPUT_DIR\" --run-id example --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --trace", + "reproducibility_shell": "powershell", + "sdk_result_availability": { + "aggregate_validation_result": false, + "full_per_case_validation_delta": true, + "full_train_eval_result": true + }, + "sdk_result_summary": { + "backend": "fake", + "baseline_prompt_id": "baseline", + "failure_summary": { + "attribution_accuracy": 1.0, + "by_category": { + "format_violation": 2 + }, + "by_prompt_split": { + "baseline:train": { + "format_violation": 2 + } + }, + "examples": [ + { + "case_id": "train_json_refund", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "train" + }, + { + "case_id": "train_exact_order_status", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "train" + } + ], + "expected_labeled_failures": 2, + "total_failed_cases": 2 + }, + "proposal_duration_seconds": 0.00011140000424347818, + "round_duration_allocation": "equal_share_of_batch_proposal_duration" + }, "seed": 91, - "total_run_cost": 0.018 + "total_run_cost": 0.018, + "total_run_cost_complete": true, + "writeback_journal": { + "after_hashes": {}, + "before_hashes": { + "system_prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95" + }, + "candidate_hashes": { + "system_prompt": "920ffe712cf686259e940bfe712fe1cf819cbeade662639576eeca12c9464f17" + }, + "error": null, + "input_hashes": { + "optimizer": "3d8c9e3f229f3ef2627dc89586952367cc456631a4b4ba20dc0313ec4098c095", + "train": "269e76792ff6fc2ce91c3a1008095dd020f4769d8af0b98d9f9410cc179a276e", + "validation": "292ee54dec4d33f34f3218672af3c56f4b3be2149960e19868f0de37e48e272a" + }, + "report_phase": "final", + "requested": false, + "run_id": "example", + "selected_candidate": "candidate_002_safe", + "state": "not_requested" + } }, "baseline": { "train": { @@ -58,55 +208,41 @@ "failure_category": "format_violation", "failure_reason": "output is not valid JSON", "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, "output": "Here is the JSON you requested: {\"intent\": \"refund\", \"priority\": \"high\"}", "passed": false, "score": 0.0, "split": "train", "trace": { - "case_id": "train_json_refund", - "judge": { - "expectation_type": "json", - "valid_json": false - }, - "model": { - "case_id": "train_json_refund", - "expectation_type": "json", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true }, { "case_id": "train_exact_order_status", "cost": 0.001, - "evidence": "expected normalized 'READY', got 'READY - confirmed.'", - "expected_failure_category": "final_response_mismatch", - "failure_category": "final_response_mismatch", - "failure_reason": "normalized exact answer mismatch", + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", "hard_failed": true, - "output": "READY - confirmed.", + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "Here is the JSON you requested: {\"next_step\": \"ship\", \"status\": \"READY\"}", "passed": false, "score": 0.0, "split": "train", "trace": { - "case_id": "train_exact_order_status", - "judge": { - "expectation_type": "exact", - "expected": "READY" - }, - "model": { - "case_id": "train_exact_order_status", - "expectation_type": "exact", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true }, { "case_id": "train_rubric_retry_summary", @@ -116,25 +252,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "latency retries", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Latency can trigger retries.", "passed": true, "score": 1.0, "split": "train", "trace": { - "case_id": "train_rubric_retry_summary", - "judge": { - "expectation_type": "rubric" - }, - "model": { - "case_id": "train_rubric_retry_summary", - "expectation_type": "rubric", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true } ], "cost": 0.003, @@ -153,26 +283,19 @@ "failure_category": "format_violation", "failure_reason": "output is not valid JSON", "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, "output": "Here is the JSON you requested: {\"next_step\": \"email_customer\", \"status\": \"approved\"}", "passed": false, "score": 0.0, "split": "validation", "trace": { - "case_id": "val_json_invoice", - "judge": { - "expectation_type": "json", - "valid_json": false - }, - "model": { - "case_id": "val_json_invoice", - "expectation_type": "json", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true }, { "case_id": "val_explain_cache", @@ -182,25 +305,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "cache stale data", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Cache invalidation refreshes stale data.", "passed": true, "score": 1.0, "split": "validation", "trace": { - "case_id": "val_explain_cache", - "judge": { - "expectation_type": "rubric" - }, - "model": { - "case_id": "val_explain_cache", - "expectation_type": "rubric", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true }, { "case_id": "val_protected_yes_no", @@ -210,26 +327,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, "output": "YES", "passed": true, "score": 1.0, "split": "validation", "trace": { - "case_id": "val_protected_yes_no", - "judge": { - "expectation_type": "exact", - "expected": "YES" - }, - "model": { - "case_id": "val_protected_yes_no", - "expectation_type": "exact", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true } ], "cost": 0.003, @@ -239,6 +349,9 @@ "split": "validation" } }, + "baseline_prompts": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n" + }, "baseline_train": { "cases": [ { @@ -249,55 +362,41 @@ "failure_category": "format_violation", "failure_reason": "output is not valid JSON", "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, "output": "Here is the JSON you requested: {\"intent\": \"refund\", \"priority\": \"high\"}", "passed": false, "score": 0.0, "split": "train", "trace": { - "case_id": "train_json_refund", - "judge": { - "expectation_type": "json", - "valid_json": false - }, - "model": { - "case_id": "train_json_refund", - "expectation_type": "json", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true }, { "case_id": "train_exact_order_status", "cost": 0.001, - "evidence": "expected normalized 'READY', got 'READY - confirmed.'", - "expected_failure_category": "final_response_mismatch", - "failure_category": "final_response_mismatch", - "failure_reason": "normalized exact answer mismatch", + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", "hard_failed": true, - "output": "READY - confirmed.", + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "Here is the JSON you requested: {\"next_step\": \"ship\", \"status\": \"READY\"}", "passed": false, "score": 0.0, "split": "train", "trace": { - "case_id": "train_exact_order_status", - "judge": { - "expectation_type": "exact", - "expected": "READY" - }, - "model": { - "case_id": "train_exact_order_status", - "expectation_type": "exact", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true }, { "case_id": "train_rubric_retry_summary", @@ -307,25 +406,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "latency retries", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Latency can trigger retries.", "passed": true, "score": 1.0, "split": "train", "trace": { - "case_id": "train_rubric_retry_summary", - "judge": { - "expectation_type": "rubric" - }, - "model": { - "case_id": "train_rubric_retry_summary", - "expectation_type": "rubric", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true } ], "cost": 0.003, @@ -344,26 +437,19 @@ "failure_category": "format_violation", "failure_reason": "output is not valid JSON", "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, "output": "Here is the JSON you requested: {\"next_step\": \"email_customer\", \"status\": \"approved\"}", "passed": false, "score": 0.0, "split": "validation", "trace": { - "case_id": "val_json_invoice", - "judge": { - "expectation_type": "json", - "valid_json": false - }, - "model": { - "case_id": "val_json_invoice", - "expectation_type": "json", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true }, { "case_id": "val_explain_cache", @@ -373,25 +459,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "cache stale data", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Cache invalidation refreshes stale data.", "passed": true, "score": 1.0, "split": "validation", "trace": { - "case_id": "val_explain_cache", - "judge": { - "expectation_type": "rubric" - }, - "model": { - "case_id": "val_explain_cache", - "expectation_type": "rubric", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true }, { "case_id": "val_protected_yes_no", @@ -401,26 +481,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, "output": "YES", "passed": true, "score": 1.0, "split": "validation", "trace": { - "case_id": "val_protected_yes_no", - "judge": { - "expectation_type": "exact", - "expected": "YES" - }, - "model": { - "case_id": "val_protected_yes_no", - "expectation_type": "exact", - "prompt_id": "baseline", - "prompt_mode": "baseline", - "seed": 91 - }, "prompt_id": "baseline", - "trace_mode": "fake" - } + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true } ], "cost": 0.003, @@ -433,9 +506,15 @@ { "candidate": { "candidate_id": "candidate_001_overfit", - "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", - "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", - "rationale": "The train failures are strict JSON/exact formatting failures, so this candidate over-corrects by forcing JSON globally." + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,5 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+Always force every final answer into JSON.", + "prompt_fields": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n" + }, + "rationale": "Observed training failures (format_violation); this candidate deliberately tests the risky global-JSON correction." + }, + "prompt_bundle": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n" }, "train_result": { "cases": [ @@ -447,55 +526,41 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, "output": "{\"intent\": \"refund\", \"priority\": \"high\"}", "passed": true, "score": 1.0, "split": "train", "trace": { - "case_id": "train_json_refund", - "judge": { - "expectation_type": "json", - "valid_json": true - }, - "model": { - "case_id": "train_json_refund", - "expectation_type": "json", - "prompt_id": "candidate_001_overfit", - "prompt_mode": "overfit", - "seed": 91 - }, "prompt_id": "candidate_001_overfit", - "trace_mode": "fake" - } + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true }, { "case_id": "train_exact_order_status", "cost": 0.001, "evidence": null, - "expected_failure_category": "final_response_mismatch", + "expected_failure_category": "format_violation", "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "READY", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"next_step\": \"ship\", \"status\": \"READY\"}", "passed": true, "score": 1.0, "split": "train", "trace": { - "case_id": "train_exact_order_status", - "judge": { - "expectation_type": "exact", - "expected": "READY" - }, - "model": { - "case_id": "train_exact_order_status", - "expectation_type": "exact", - "prompt_id": "candidate_001_overfit", - "prompt_mode": "overfit", - "seed": 91 - }, "prompt_id": "candidate_001_overfit", - "trace_mode": "fake" - } + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true }, { "case_id": "train_rubric_retry_summary", @@ -505,25 +570,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "latency retries", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"answer\": \"Latency can trigger retries.\"}", "passed": true, "score": 1.0, "split": "train", "trace": { - "case_id": "train_rubric_retry_summary", - "judge": { - "expectation_type": "rubric" - }, - "model": { - "case_id": "train_rubric_retry_summary", - "expectation_type": "rubric", - "prompt_id": "candidate_001_overfit", - "prompt_mode": "overfit", - "seed": 91 - }, "prompt_id": "candidate_001_overfit", - "trace_mode": "fake" - } + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true } ], "cost": 0.003, @@ -542,26 +601,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, "output": "{\"next_step\": \"email_customer\", \"status\": \"approved\"}", "passed": true, "score": 1.0, "split": "validation", "trace": { - "case_id": "val_json_invoice", - "judge": { - "expectation_type": "json", - "valid_json": true - }, - "model": { - "case_id": "val_json_invoice", - "expectation_type": "json", - "prompt_id": "candidate_001_overfit", - "prompt_mode": "overfit", - "seed": 91 - }, "prompt_id": "candidate_001_overfit", - "trace_mode": "fake" - } + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true }, { "case_id": "val_explain_cache", @@ -571,26 +623,19 @@ "failure_category": "format_violation", "failure_reason": "output contains a forbidden pattern", "hard_failed": true, - "output": "{\"answer\": \"cache stale data\"}", + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "{\"answer\": \"Cache invalidation refreshes stale data.\"}", "passed": false, "score": 0.0, "split": "validation", "trace": { - "case_id": "val_explain_cache", - "judge": { - "expectation_type": "rubric", - "forbidden_pattern": "{" - }, - "model": { - "case_id": "val_explain_cache", - "expectation_type": "rubric", - "prompt_id": "candidate_001_overfit", - "prompt_mode": "overfit", - "seed": 91 - }, "prompt_id": "candidate_001_overfit", - "trace_mode": "fake" - } + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true }, { "case_id": "val_protected_yes_no", @@ -600,26 +645,19 @@ "failure_category": "final_response_mismatch", "failure_reason": "normalized exact answer mismatch", "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, "output": "{\"answer\": \"YES\"}", "passed": false, "score": 0.0, "split": "validation", "trace": { - "case_id": "val_protected_yes_no", - "judge": { - "expectation_type": "exact", - "expected": "YES" - }, - "model": { - "case_id": "val_protected_yes_no", - "expectation_type": "exact", - "prompt_id": "candidate_001_overfit", - "prompt_mode": "overfit", - "seed": 91 - }, "prompt_id": "candidate_001_overfit", - "trace_mode": "fake" - } + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true } ], "cost": 0.003, @@ -632,9 +670,15 @@ { "candidate": { "candidate_id": "candidate_002_safe", - "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\nUse strict JSON only when the user explicitly asks for JSON.\nUse exact answers only when the user explicitly asks for an exact answer.\nOtherwise answer naturally and honor rubric constraints.\n", - "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints.", - "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,5 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+Use strict JSON only when the user explicitly asks.", + "prompt_fields": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n" + }, + "rationale": "Observed training failures (format_violation); this candidate limits strict JSON behavior to explicit user requests." + }, + "prompt_bundle": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n" }, "train_result": { "cases": [ @@ -646,55 +690,41 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, "output": "{\"intent\": \"refund\", \"priority\": \"high\"}", "passed": true, "score": 1.0, "split": "train", "trace": { - "case_id": "train_json_refund", - "judge": { - "expectation_type": "json", - "valid_json": true - }, - "model": { - "case_id": "train_json_refund", - "expectation_type": "json", - "prompt_id": "candidate_002_safe", - "prompt_mode": "safe", - "seed": 91 - }, "prompt_id": "candidate_002_safe", - "trace_mode": "fake" - } + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true }, { "case_id": "train_exact_order_status", "cost": 0.001, "evidence": null, - "expected_failure_category": "final_response_mismatch", + "expected_failure_category": "format_violation", "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "READY", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"next_step\": \"ship\", \"status\": \"READY\"}", "passed": true, "score": 1.0, "split": "train", "trace": { - "case_id": "train_exact_order_status", - "judge": { - "expectation_type": "exact", - "expected": "READY" - }, - "model": { - "case_id": "train_exact_order_status", - "expectation_type": "exact", - "prompt_id": "candidate_002_safe", - "prompt_mode": "safe", - "seed": 91 - }, "prompt_id": "candidate_002_safe", - "trace_mode": "fake" - } + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true }, { "case_id": "train_rubric_retry_summary", @@ -704,25 +734,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "latency retries", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Latency can trigger retries.", "passed": true, "score": 1.0, "split": "train", "trace": { - "case_id": "train_rubric_retry_summary", - "judge": { - "expectation_type": "rubric" - }, - "model": { - "case_id": "train_rubric_retry_summary", - "expectation_type": "rubric", - "prompt_id": "candidate_002_safe", - "prompt_mode": "safe", - "seed": 91 - }, "prompt_id": "candidate_002_safe", - "trace_mode": "fake" - } + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true } ], "cost": 0.003, @@ -741,26 +765,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, "output": "{\"next_step\": \"email_customer\", \"status\": \"approved\"}", "passed": true, "score": 1.0, "split": "validation", "trace": { - "case_id": "val_json_invoice", - "judge": { - "expectation_type": "json", - "valid_json": true - }, - "model": { - "case_id": "val_json_invoice", - "expectation_type": "json", - "prompt_id": "candidate_002_safe", - "prompt_mode": "safe", - "seed": 91 - }, "prompt_id": "candidate_002_safe", - "trace_mode": "fake" - } + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true }, { "case_id": "val_explain_cache", @@ -770,25 +787,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, - "output": "cache stale data", + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Cache invalidation refreshes stale data.", "passed": true, "score": 1.0, "split": "validation", "trace": { - "case_id": "val_explain_cache", - "judge": { - "expectation_type": "rubric" - }, - "model": { - "case_id": "val_explain_cache", - "expectation_type": "rubric", - "prompt_id": "candidate_002_safe", - "prompt_mode": "safe", - "seed": 91 - }, "prompt_id": "candidate_002_safe", - "trace_mode": "fake" - } + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true }, { "case_id": "val_protected_yes_no", @@ -798,26 +809,19 @@ "failure_category": null, "failure_reason": null, "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, "output": "YES", "passed": true, "score": 1.0, "split": "validation", "trace": { - "case_id": "val_protected_yes_no", - "judge": { - "expectation_type": "exact", - "expected": "YES" - }, - "model": { - "case_id": "val_protected_yes_no", - "expectation_type": "exact", - "prompt_id": "candidate_002_safe", - "prompt_mode": "safe", - "seed": 91 - }, "prompt_id": "candidate_002_safe", - "trace_mode": "fake" - } + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true } ], "cost": 0.003, @@ -828,6 +832,14 @@ } } ], + "cost_summary": { + "agent": 0.0, + "complete": true, + "evaluator": 0.018, + "optimizer": 0.0, + "reported_optimizer_cost": null, + "total": 0.018 + }, "delta": { "per_case": [ { @@ -979,13 +991,12 @@ "failure_attribution_summary": { "attribution_accuracy": 1.0, "by_category": { - "final_response_mismatch": 2, - "format_violation": 3 + "final_response_mismatch": 1, + "format_violation": 4 }, "by_prompt_split": { "baseline:train": { - "final_response_mismatch": 1, - "format_violation": 1 + "format_violation": 2 }, "baseline:validation": { "format_violation": 1 @@ -1009,9 +1020,9 @@ }, { "case_id": "train_exact_order_status", - "evidence": "expected normalized 'READY', got 'READY - confirmed.'", - "failure_category": "final_response_mismatch", - "failure_reason": "normalized exact answer mismatch", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", "prompt_id": "baseline", "split": "train" }, @@ -1068,6 +1079,7 @@ "reasons": [ "reject: overfit detected because train score improved but validation score regressed or did not improve (+0.667 train, -0.333 validation)", "reject: validation improvement -0.333 is below required +0.010", + "reject: new validation failures appeared: ['val_explain_cache', 'val_protected_yes_no']", "reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no']", "reject: protected cases regressed: ['val_protected_yes_no']", "reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no']" @@ -1248,24 +1260,75 @@ "split": "validation" } ], + "rounds": [ + { + "candidate_id": "candidate_001_overfit", + "cost": { + "agent": 0.0, + "complete": true, + "evaluator": 0.0, + "optimizer": 0.0, + "reported_optimizer_cost": null, + "total": 0.0 + }, + "duration_seconds": 5.570000212173909e-05, + "metrics": {}, + "prompts": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n" + }, + "rationale": "Observed training failures (format_violation); this candidate deliberately tests the risky global-JSON correction.", + "round_id": 1 + }, + { + "candidate_id": "candidate_002_safe", + "cost": { + "agent": 0.0, + "complete": true, + "evaluator": 0.0, + "optimizer": 0.0, + "reported_optimizer_cost": null, + "total": 0.0 + }, + "duration_seconds": 5.570000212173909e-05, + "metrics": {}, + "prompts": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n" + }, + "rationale": "Observed training failures (format_violation); this candidate limits strict JSON behavior to explicit user requests.", + "round_id": 2 + } + ], "run": { - "fake_judge": true, - "fake_model": true, "mode": "fake", "paths": { - "optimizer": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\optimizer.json", - "prompt": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", - "train": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\train.evalset.json", - "validation": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\val.evalset.json" + "optimizer": "examples/optimization/eval_optimize_loop/data/optimizer.json", + "prompts": { + "system_prompt": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt" + }, + "train": "examples/optimization/eval_optimize_loop/data/train.evalset.json", + "validation": "examples/optimization/eval_optimize_loop/data/val.evalset.json" + }, + "prompt_source": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt", + "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --output-dir \"$OUTPUT_DIR\" --run-id example --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --trace", + "reproducibility_shell": "powershell", + "run_id": "example", + "target_prompt_paths": { + "system_prompt": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt" }, - "prompt_source": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", - "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", - "run_id": "eval_optimize_loop_seed_91", + "trace": true, "trace_enabled": true, "train_cases": 3, "update_source": false, "validation_cases": 3 }, - "schema_version": "eval_optimize_loop.v1", - "selected_candidate": "candidate_002_safe" + "schema_version": "eval_optimize_loop.v2", + "selected_candidate": "candidate_002_safe", + "writeback": { + "after_hashes": {}, + "before_hashes": { + "system_prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95" + }, + "error": null, + "status": "not_requested" + } } diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md index 510c7f65..7380b319 100644 --- a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md @@ -6,12 +6,15 @@ Selected candidate: `candidate_002_safe`. Update source prompt: no (default) +Fake and SDK modes perform complete AgentEvaluator-compatible reevaluation for baseline and every candidate on both train and validation; optimizer aggregates are never used as gate evidence. + ## Gate Reasons ### candidate_001_overfit (rejected) - reject: overfit detected because train score improved but validation score regressed or did not improve (+0.667 train, -0.333 validation) - reject: validation improvement -0.333 is below required +0.010 +- reject: new validation failures appeared: ['val_explain_cache', 'val_protected_yes_no'] - reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no'] - reject: protected cases regressed: ['val_protected_yes_no'] - reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no'] @@ -50,16 +53,16 @@ Total failed case evaluations: 5 | category | count | | --- | ---: | -| final_response_mismatch | 2 | -| format_violation | 3 | +| final_response_mismatch | 1 | +| format_violation | 4 | Attribution accuracy: 1.000 ## Cost And Audit Total cost: 0.018 -Config hash: `66a1db3a1c84ad12fd41385fc5e1a9c23cc305e6973945c4bade559c804f9abe` -Run id: `eval_optimize_loop_seed_91` +Config hash: `7144548d6c99b0a6226534dfe51983875c26bf3193b7ad07aaecf9a1cb80dcfd` +Run id: `example` ## Prompt Diff @@ -68,14 +71,12 @@ Run id: `eval_optimize_loop_seed_91` ```diff --- baseline_system_prompt.txt +++ candidate_001_overfit/system_prompt.txt -@@ -2,3 +2,7 @@ +@@ -2,3 +2,5 @@ Answer clearly and include a short explanation when it may help the user. If the user asks for structured data, provide the information they need. + -+# Optimizer patch -+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON -+Always force every final answer into JSON, even when the user asks for prose. ++Always force every final answer into JSON. ``` ### candidate_002_safe @@ -83,20 +84,16 @@ Run id: `eval_optimize_loop_seed_91` ```diff --- baseline_system_prompt.txt +++ candidate_002_safe/system_prompt.txt -@@ -2,3 +2,9 @@ +@@ -2,3 +2,5 @@ Answer clearly and include a short explanation when it may help the user. If the user asks for structured data, provide the information they need. + -+# Optimizer patch -+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED -+Use strict JSON only when the user explicitly asks for JSON. -+Use exact answers only when the user explicitly asks for an exact answer. -+Otherwise answer naturally and honor rubric constraints. ++Use strict JSON only when the user explicitly asks. ``` ## Reproducibility -```bash -python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace +```powershell +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --output-dir "$OUTPUT_DIR" --run-id example --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --trace ``` diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index be0dbe33..64ab737c 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -4,7 +4,6 @@ import argparse import asyncio -import json import re import secrets import sys @@ -14,7 +13,6 @@ from pathlib import Path from typing import Any - HERE = Path(__file__).resolve().parent if str(HERE) not in sys.path: sys.path.insert(0, str(HERE)) @@ -44,7 +42,6 @@ from eval_loop.pipeline import execute_pipeline from eval_loop.schemas import OptimizationReport - DEFAULT_TRAIN = HERE / "data" / "train.evalset.json" DEFAULT_VAL = HERE / "data" / "val.evalset.json" DEFAULT_OPTIMIZER_CONFIG = HERE / "data" / "optimizer.json" @@ -119,10 +116,8 @@ def run_pipeline( """Synchronous compatibility facade for callers without an active loop.""" if _has_running_loop(): - raise ValueError( - "run_pipeline() cannot run while an event loop is active; " - "await run_pipeline_async(...) instead." - ) + raise ValueError("run_pipeline() cannot run while an event loop is active; " + "await run_pipeline_async(...) instead.") return asyncio.run( run_pipeline_async( train_path=train_path, @@ -140,8 +135,7 @@ def run_pipeline( target_prompts=target_prompts, run_id=run_id, backend=backend, - ) - ) + )) def build_pipeline_request_and_backend( @@ -167,13 +161,14 @@ def build_pipeline_request_and_backend( if mode not in {"fake", "sdk"}: raise ValueError("field 'mode' must be one of: fake, sdk") if mode == "fake" and (not fake_model or not fake_judge): - raise ValueError( - "fake mode requires fake_model=True and fake_judge=True. Pass --fake-model " - "--fake-judge or use --mode sdk with --sdk-call-agent module:function." - ) + raise ValueError("fake mode requires fake_model=True and fake_judge=True. Pass --fake-model " + "--fake-judge or use --mode sdk with --sdk-call-agent module:function.") validate_distinct_file_paths( - {"train": train_path, "validation": val_path}, + { + "train": train_path, + "validation": val_path + }, context="train and validation evalset paths", ) @@ -191,24 +186,15 @@ def build_pipeline_request_and_backend( ) if mode == "fake" and backend is not None and hasattr(backend, "seed"): backend_seed = getattr(backend, "seed") - if ( - isinstance(backend_seed, bool) - or not isinstance(backend_seed, int) - or backend_seed != effective_seed - ): - raise ValueError( - f"fake backend seed {backend_seed!r} does not match effective seed {effective_seed}" - ) + if (isinstance(backend_seed, bool) or not isinstance(backend_seed, int) or backend_seed != effective_seed): + raise ValueError(f"fake backend seed {backend_seed!r} does not match effective seed {effective_seed}") if mode == "fake": optimizer_config = parse_optimizer_config( optimizer_payload, path=optimizer_config_path, ) - gate_config = ( - _load_sdk_gate_config(gate_config_path) - if gate_config_path is not None - else optimizer_config.gate.to_dict() - ) + gate_config = (_load_sdk_gate_config(gate_config_path) + if gate_config_path is not None else optimizer_config.gate.to_dict()) gate_config_source = "file" if gate_config_path is not None else "optimizer" selected_backend = backend or FakeBackend( seed=effective_seed, @@ -257,10 +243,8 @@ def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, Any] try: payload = read_json(gate_config_path) except ValueError as exc: - raise ValueError( - f"--gate-config {gate_config_path}: invalid JSON for gate numeric fields " - f"(including max_total_cost): {exc}" - ) from exc + raise ValueError(f"--gate-config {gate_config_path}: invalid JSON for gate numeric fields " + f"(including max_total_cost): {exc}") from exc gate_payload = payload.get("gate", payload) path_text = str(gate_config_path) if gate_payload is None: @@ -290,10 +274,8 @@ def _parse_target_prompt_paths( name, raw_path = item.split("=", 1) path = raw_path.strip() if not TARGET_PROMPT_FIELD_RE.fullmatch(name): - raise ValueError( - f"--target-prompt field name {name!r} is invalid; " - "use /^[A-Za-z_][A-Za-z0-9_]*$/" - ) + raise ValueError(f"--target-prompt field name {name!r} is invalid; " + "use /^[A-Za-z_][A-Za-z0-9_]*$/") try: validate_artifact_component(name, context="target-prompt field") except ValueError as error: diff --git a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py index c96bafa3..2753c688 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py +++ b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py @@ -47,15 +47,11 @@ def test_optimizer_config_rejects_negative_cost(tmp_path: Path): @pytest.mark.parametrize( ("field_name", "field_value"), - [ - (field_name, field_value) - for field_name in ( - "min_val_score_improvement", - "max_score_drop_per_case", - "max_total_cost", - ) - for field_value in (float("nan"), float("inf"), float("-inf"), True) - ], + [(field_name, field_value) for field_name in ( + "min_val_score_improvement", + "max_score_drop_per_case", + "max_total_cost", + ) for field_value in (float("nan"), float("inf"), float("-inf"), True)], ) def test_optimizer_config_rejects_non_finite_or_boolean_gate_numbers( tmp_path: Path, @@ -70,7 +66,9 @@ def test_optimizer_config_rejects_non_finite_or_boolean_gate_numbers( def test_optimizer_config_allows_disabling_cost_gate(tmp_path: Path): config = parse_optimizer_config( - {"gate": {"max_total_cost": None}}, + {"gate": { + "max_total_cost": None + }}, path=tmp_path / "optimizer.json", ) @@ -256,7 +254,10 @@ def test_eval_case_rejects_explicit_split_mismatch(): "id": "case_1", "split": "validation", "input": "hello", - "expectation": {"type": "exact", "expected": "hello"}, + "expectation": { + "type": "exact", + "expected": "hello" + }, } with pytest.raises(ValueError, match="split mismatch"): @@ -461,7 +462,9 @@ def test_validate_inputs_rejects_missing_protected_case(tmp_path: Path): train_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) val_path = _write_evalset(tmp_path / "val.evalset.json", "validation", ["v1", "v2", "v3"]) config = parse_optimizer_config( - {"gate": {"protected_case_ids": ["missing"]}}, + {"gate": { + "protected_case_ids": ["missing"] + }}, path=tmp_path / "optimizer.json", ) @@ -478,15 +481,19 @@ def test_validate_inputs_rejects_missing_protected_case(tmp_path: Path): def _write_evalset(path: Path, split: str, ids: list[str]) -> Path: payload = { - "split": split, - "cases": [ - { - "id": case_id, - "input": "Return JSON", - "expectation": {"type": "json", "required_keys": ["answer"], "expected_values": {"answer": case_id}}, - } - for case_id in ids - ], + "split": + split, + "cases": [{ + "id": case_id, + "input": "Return JSON", + "expectation": { + "type": "json", + "required_keys": ["answer"], + "expected_values": { + "answer": case_id + } + }, + } for case_id in ids], } path.write_text(json.dumps(payload), encoding="utf-8") return path diff --git a/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py index 60cbeaf5..b7175a06 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py +++ b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py @@ -22,14 +22,19 @@ def test_evaluator_attaches_failure_details_to_failed_cases(): expectation={ "type": "json", "required_keys": ["answer"], - "expected_values": {"answer": "ok"}, + "expected_values": { + "answer": "ok" + }, }, ), EvalCase( case_id="exact_case", split="train", input="Answer exactly OK.", - expectation={"type": "exact", "expected": "OK"}, + expectation={ + "type": "exact", + "expected": "OK" + }, ), ] result = ExampleEvaluator(FakeModel(seed=91), FakeJudge()).evaluate( @@ -60,7 +65,13 @@ def test_fake_judge_scores_tool_and_parameter_expectations(): case_id="tool_case", split="train", input="Call tool", - expectation={"type": "tool", "expected_tool": "lookup", "expected_args": {"id": "42"}}, + expectation={ + "type": "tool", + "expected_tool": "lookup", + "expected_args": { + "id": "42" + } + }, ) assert judge.score(tool_case, '{"tool": "lookup", "args": {"id": "42"}}').passed @@ -97,7 +108,12 @@ def test_failure_summary_computes_attribution_accuracy(): case_id="json_labeled", split="train", input="Return JSON", - expectation={"type": "json", "expected_values": {"answer": "ok"}}, + expectation={ + "type": "json", + "expected_values": { + "answer": "ok" + } + }, expected_failure_category="format_violation", ) ], diff --git a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py index c8517743..f6caa0a4 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py +++ b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py @@ -33,22 +33,25 @@ def test_model_output_and_trace_are_stable_for_the_same_public_input(): public_input = "Return strict JSON with intent=refund and priority=high." same_inputs = [public_input, f"{public_input}", "".join([public_input])] - generated = [ - FakeModel(seed=91).generate("candidate", SAFE_PROMPT, user_input) - for user_input in same_inputs - ] + generated = [FakeModel(seed=91).generate("candidate", SAFE_PROMPT, user_input) for user_input in same_inputs] assert {output for output, _, _ in generated} == {'{"intent": "refund", "priority": "high"}'} - assert {json.dumps(trace, sort_keys=True) for _, trace, _ in generated} == { - json.dumps( - {"seed": 91, "prompt_id": "candidate", "prompt_mode": "safe"}, - sort_keys=True, - ) - } + assert {json.dumps(trace, sort_keys=True) + for _, trace, _ in generated + } == {json.dumps( + { + "seed": 91, + "prompt_id": "candidate", + "prompt_mode": "safe" + }, + sort_keys=True, + )} def test_example_evaluator_only_exposes_public_user_input_to_model(): + class SpyModel: + def __init__(self) -> None: self.user_inputs: list[str] = [] @@ -67,7 +70,10 @@ def generate( case_id="SECRET_CASE_ID", split="validation", input="PUBLIC USER INPUT", - expectation={"type": "exact", "expected": "SECRET_EXPECTATION"}, + expectation={ + "type": "exact", + "expected": "SECRET_EXPECTATION" + }, tags=["SECRET_TAG"], protected=True, simulated_outputs={"safe": "SECRET_SIMULATED_OUTPUT"}, @@ -180,45 +186,35 @@ def test_natural_and_unknown_requests_are_deterministic(input_text: str, natural def test_optimizer_returns_no_candidates_without_observed_target_failures(): optimizer = FakeOptimizer() - all_pass = _eval_result( - _case_result( - "passed_case", - passed=True, - failure_category="format_violation", - ) - ) - unrelated_failure = _eval_result( - _case_result( - "failed_case", - passed=False, - failure_category="llm_rubric_not_met", - ) - ) - unclassified_failure = _eval_result( - _case_result( - "unclassified_failure", - passed=False, - failure_category=None, - ) - ) - - assert ( - optimizer.propose( - BASELINE_PROMPT, - all_pass, - {}, - ) - == [] - ) + all_pass = _eval_result(_case_result( + "passed_case", + passed=True, + failure_category="format_violation", + )) + unrelated_failure = _eval_result(_case_result( + "failed_case", + passed=False, + failure_category="llm_rubric_not_met", + )) + unclassified_failure = _eval_result(_case_result( + "unclassified_failure", + passed=False, + failure_category=None, + )) + + assert (optimizer.propose( + BASELINE_PROMPT, + all_pass, + {}, + ) == []) assert optimizer.propose(BASELINE_PROMPT, unclassified_failure, {}) == [] - assert ( - optimizer.propose( - BASELINE_PROMPT, - unrelated_failure, - {"by_category": {"llm_rubric_not_met": 1}}, - ) - == [] - ) + assert (optimizer.propose( + BASELINE_PROMPT, + unrelated_failure, + {"by_category": { + "llm_rubric_not_met": 1 + }}, + ) == []) def test_optimizer_rejects_validation_result_as_training_evidence(): @@ -258,16 +254,13 @@ def test_optimizer_rejects_validation_case_mixed_into_training_evidence(): @pytest.mark.parametrize("failure_summary", [None, [], "not-a-summary"]) -def test_optimizer_rejects_non_mapping_failure_summary( - failure_summary: object, -): +def test_optimizer_rejects_non_mapping_failure_summary(failure_summary: object, ): observed_failure = _eval_result( _case_result( "observed_failure", passed=False, failure_category="format_violation", - ) - ) + )) with pytest.raises(TypeError, match="^failure_summary must be a dict$"): FakeOptimizer().propose( @@ -278,40 +271,41 @@ def test_optimizer_rejects_non_mapping_failure_summary( def test_optimizer_rejects_summary_category_not_observed_in_failed_cases(): - unclassified_failure = _eval_result( - _case_result( - "unclassified_failure", - passed=False, - failure_category=None, - ) - ) + unclassified_failure = _eval_result(_case_result( + "unclassified_failure", + passed=False, + failure_category=None, + )) with pytest.raises(ValueError, match="by_category.*failed train cases"): FakeOptimizer().propose( BASELINE_PROMPT, unclassified_failure, - {"by_category": {"format_violation": 1}}, + {"by_category": { + "format_violation": 1 + }}, ) @pytest.mark.parametrize( "by_category", [ - {"final_response_mismatch": 1}, - {"format_violation": 2}, + { + "final_response_mismatch": 1 + }, + { + "format_violation": 2 + }, {}, ], ) -def test_optimizer_rejects_summary_category_or_count_mismatch( - by_category: dict[str, object], -): +def test_optimizer_rejects_summary_category_or_count_mismatch(by_category: dict[str, object], ): observed_failure = _eval_result( _case_result( "observed_failure", passed=False, failure_category="format_violation", - ) - ) + )) with pytest.raises(ValueError, match="by_category.*failed train cases"): FakeOptimizer().propose( @@ -338,14 +332,15 @@ def test_optimizer_rejects_non_positive_integer_summary_counts(count: object): "observed_failure", passed=False, failure_category="format_violation", - ) - ) + )) with pytest.raises(ValueError, match="count.*positive integer"): FakeOptimizer().propose( BASELINE_PROMPT, observed_failure, - {"by_category": {"format_violation": count}}, + {"by_category": { + "format_violation": count + }}, ) @@ -353,19 +348,20 @@ def test_optimizer_rejects_non_positive_integer_summary_counts(count: object): "failure_summary", [ {}, - {"by_category": {"format_violation": 1}}, + { + "by_category": { + "format_violation": 1 + } + }, ], ) -def test_optimizer_proposes_two_candidates_for_observed_train_format_failure( - failure_summary: dict[str, object], -): +def test_optimizer_proposes_two_candidates_for_observed_train_format_failure(failure_summary: dict[str, object], ): baseline_train = _eval_result( _case_result( "random_training_case", passed=False, failure_category="format_violation", - ) - ) + )) candidates = FakeOptimizer().propose( BASELINE_PROMPT, baseline_train, @@ -398,14 +394,14 @@ async def test_fake_backend_passes_training_evidence_to_optimizer(tmp_path: Path ) with_evidence = await backend.optimize_candidates( baseline_prompts={"system_prompt": BASELINE_PROMPT}, - baseline_train=_eval_result( - _case_result( - "not-a-sample-id", - passed=False, - failure_category="format_violation", - ) - ), - failure_summary={"by_category": {"format_violation": 1}}, + baseline_train=_eval_result(_case_result( + "not-a-sample-id", + passed=False, + failure_category="format_violation", + )), + failure_summary={"by_category": { + "format_violation": 1 + }}, train_path=TRAIN_PATH, validation_path=VALIDATION_PATH, config_path=tmp_path / "unused.json", @@ -449,23 +445,19 @@ def test_example_data_is_official_sdk_evalset_with_self_contained_user_inputs(): raw = path.read_text(encoding="utf-8") validated = EvalSet.model_validate_json(raw) payload = json.loads(raw) - assert [ - case["conversation"][-1]["user_content"]["parts"][0]["text"] for case in payload["eval_cases"] - ] == inputs + assert [case["conversation"][-1]["user_content"]["parts"][0]["text"] + for case in payload["eval_cases"]] == inputs assert len(validated.eval_cases) == 3 all_eval_ids.extend(case.eval_id for case in validated.eval_cases) - all_invocation_ids.extend( - invocation.invocation_id for case in validated.eval_cases for invocation in case.conversation or [] - ) + all_invocation_ids.extend(invocation.invocation_id for case in validated.eval_cases + for invocation in case.conversation or []) assert len(all_eval_ids) == len(set(all_eval_ids)) == 6 assert len(all_invocation_ids) == len(set(all_invocation_ids)) == 6 @pytest.mark.asyncio -async def test_fake_backend_scores_baseline_overfit_and_safe_on_official_data( - tmp_path: Path, -): +async def test_fake_backend_scores_baseline_overfit_and_safe_on_official_data(tmp_path: Path, ): backend = FakeBackend(seed=91) prompts = { "baseline": BASELINE_PROMPT, @@ -497,15 +489,11 @@ async def test_fake_backend_scores_baseline_overfit_and_safe_on_official_data( expected_score, expected_passes = expected[(prompt_id, split)] assert result.score == pytest.approx(expected_score, abs=1e-6) assert [case.passed for case in result.cases] == expected_passes - assert all( - case.trace - == { - "seed": 91, - "prompt_id": prompt_id, - "prompt_mode": prompt_id, - } - for case in result.cases - ) + assert all(case.trace == { + "seed": 91, + "prompt_id": prompt_id, + "prompt_mode": prompt_id, + } for case in result.cases) def _case_result( diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py index deb16a28..c6572a11 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_gate.py +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -20,7 +20,9 @@ def test_gate_rejects_train_improvement_with_validation_regression(): candidate_train=candidate_train, candidate_validation=candidate_val, ) - decision = AcceptanceGate({"protected_case_ids": []}).decide( + decision = AcceptanceGate({ + "protected_case_ids": [] + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -48,7 +50,9 @@ def test_gate_rejects_protected_case_regression(): candidate_train=candidate_train, candidate_validation=candidate_val, ) - decision = AcceptanceGate({"protected_case_ids": ["protected"]}).decide( + decision = AcceptanceGate({ + "protected_case_ids": ["protected"] + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -76,7 +80,9 @@ def test_gate_accepts_safe_candidate(): candidate_train=candidate_train, candidate_validation=candidate_val, ) - decision = AcceptanceGate({"protected_case_ids": ["protected"]}).decide( + decision = AcceptanceGate({ + "protected_case_ids": ["protected"] + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -104,7 +110,9 @@ def test_gate_rejects_new_hard_failure_when_not_allowed(): candidate_validation=candidate_val, ) - decision = AcceptanceGate({"allow_new_hard_fail": False}).decide( + decision = AcceptanceGate({ + "allow_new_hard_fail": False + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -164,7 +172,10 @@ def test_gate_rejects_excessive_score_drop(): candidate_validation=candidate_val, ) - decision = AcceptanceGate({"max_score_drop_per_case": 0.5, "allow_new_hard_fail": True}).decide( + decision = AcceptanceGate({ + "max_score_drop_per_case": 0.5, + "allow_new_hard_fail": True + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -191,7 +202,9 @@ def test_gate_rejects_cost_budget(): candidate_validation=candidate_val, ) - decision = AcceptanceGate({"max_total_cost": 0.001}).decide( + decision = AcceptanceGate({ + "max_total_cost": 0.001 + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -213,7 +226,9 @@ def test_gate_rejects_incomplete_cost_when_budget_is_configured(): baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) - decision = AcceptanceGate({"max_total_cost": 100.0}).decide( + decision = AcceptanceGate({ + "max_total_cost": 100.0 + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -241,7 +256,9 @@ def test_gate_skips_cost_completeness_check_when_budget_is_disabled(): baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) - decision = AcceptanceGate({"max_total_cost": None}).decide( + decision = AcceptanceGate({ + "max_total_cost": None + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -301,7 +318,9 @@ def test_gate_does_not_add_cost_summary_total_twice(): baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) - decision = AcceptanceGate({"max_total_cost": 1.0}).decide( + decision = AcceptanceGate({ + "max_total_cost": 1.0 + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -352,7 +371,9 @@ def test_gate_rejects_duplicate_missing_and_empty_case_results(): ] for baseline_train, candidate_train, baseline_val, candidate_val, expected in scenarios: - decision = AcceptanceGate({"max_total_cost": None}).decide( + decision = AcceptanceGate({ + "max_total_cost": None + }).decide( candidate_id="candidate", baseline_train=baseline_train, baseline_validation=baseline_val, @@ -398,8 +419,7 @@ def _eval(prompt_id: str, split: str, scores: list[tuple[str, float]]) -> EvalRe passed=score >= 1.0, output="", hard_failed=score <= 0.0, - ) - for case_id, score in scores + ) for case_id, score in scores ] return EvalResult( prompt_id=prompt_id, diff --git a/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py index 16ad5fc6..a005bde1 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py +++ b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py @@ -25,7 +25,9 @@ def test_sample_case_ids_do_not_appear_in_fake_runtime_logic(): case_ids = { - case["eval_id"] for path in DATA_FILES for case in json.loads(path.read_text(encoding="utf-8"))["eval_cases"] + case["eval_id"] + for path in DATA_FILES + for case in json.loads(path.read_text(encoding="utf-8"))["eval_cases"] } for path in NO_SAMPLE_ID_FILES: diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py index 375e926c..5ed95871 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -94,10 +94,8 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert not decisions["candidate_001_overfit"]["accepted"] assert decisions["candidate_002_safe"]["accepted"] assert decisions["candidate_001_overfit"]["total_run_cost"] > decisions["candidate_001_overfit"]["candidate_cost"] - assert any( - "train score improved but validation score regressed" in reason - for reason in decisions["candidate_001_overfit"]["reasons"] - ) + assert any("train score improved but validation score regressed" in reason + for reason in decisions["candidate_001_overfit"]["reasons"]) markdown = md_path.read_text(encoding="utf-8") assert "Selected candidate: `candidate_002_safe`." in markdown @@ -118,9 +116,8 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): assert (run_dir / "candidate_prompts" / overfit_artifact / "system_prompt.txt").is_file() assert (run_dir / "case_results" / f"{safe_artifact}_validation.json").is_file() assert (run_dir / "prompt_diffs" / f"{safe_artifact}.diff").is_file() - assert (run_dir / "prompt_diffs" / f"{safe_artifact}.diff").read_text(encoding="utf-8") == ( - payload["candidates"][1]["candidate"]["prompt_diff"] - ) + assert (run_dir / "prompt_diffs" / f"{safe_artifact}.diff").read_text( + encoding="utf-8") == (payload["candidates"][1]["candidate"]["prompt_diff"]) def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): @@ -182,11 +179,14 @@ def test_cli_mode_fake_and_legacy_fake_flags_both_run(tmp_path: Path): second = tmp_path / "second" subprocess.run( - [sys.executable, str(script), "--mode", "fake", "--trace", "--output-dir", str(first)], + [sys.executable, str(script), "--mode", "fake", "--trace", "--output-dir", + str(first)], check=True, ) subprocess.run( - [sys.executable, str(script), "--fake-model", "--fake-judge", "--trace", "--output-dir", str(second)], + [sys.executable, + str(script), "--fake-model", "--fake-judge", "--trace", "--output-dir", + str(second)], check=True, ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py index 912db48a..1bb7ba4f 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py @@ -29,6 +29,7 @@ class RecordingBackend: + def __init__( self, *, @@ -81,6 +82,7 @@ async def optimize_candidates( class MutatingBundleBackend(RecordingBackend): + async def evaluate(self, **kwargs: Any) -> EvalResult: result = await super().evaluate(**kwargs) kwargs["prompts"]["system_prompt"] = "backend-mutated prompt" @@ -93,6 +95,7 @@ async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: class FailingCandidateBackend(RecordingBackend): + def __init__( self, *, @@ -110,38 +113,36 @@ def __init__( async def evaluate(self, **kwargs: Any) -> EvalResult: if (kwargs["prompt_id"], kwargs["split"]) == self.fail_on: - self.calls.append( - ( - "evaluate", - kwargs["prompt_id"], - kwargs["split"], - dict(kwargs["prompts"]), - Path(kwargs["artifact_dir"]), - kwargs["trace"], - ) - ) + self.calls.append(( + "evaluate", + kwargs["prompt_id"], + kwargs["split"], + dict(kwargs["prompts"]), + Path(kwargs["artifact_dir"]), + kwargs["trace"], + )) raise self.failure return await super().evaluate(**kwargs) class AllCandidatesFailingBackend(RecordingBackend): + async def evaluate(self, **kwargs: Any) -> EvalResult: if kwargs["prompt_id"] != "baseline": - self.calls.append( - ( - "evaluate", - kwargs["prompt_id"], - kwargs["split"], - dict(kwargs["prompts"]), - Path(kwargs["artifact_dir"]), - kwargs["trace"], - ) - ) + self.calls.append(( + "evaluate", + kwargs["prompt_id"], + kwargs["split"], + dict(kwargs["prompts"]), + Path(kwargs["artifact_dir"]), + kwargs["trace"], + )) raise RuntimeError(f"{kwargs['prompt_id']} evaluation unavailable") return await super().evaluate(**kwargs) class SourceMutatingBackend(RecordingBackend): + def __init__( self, *, @@ -194,7 +195,9 @@ async def test_execute_pipeline_uses_train_only_evidence_and_full_candidate_reev ("evaluate", "baseline", "validation"), ( "optimize", - {"system_prompt": "baseline\n"}, + { + "system_prompt": "baseline\n" + }, tmp_path / "out" / "runs" / ".ordered_run.tmp" / "optimizer", ), ("evaluate", "candidate_a", "train"), @@ -228,11 +231,7 @@ async def test_backend_cannot_mutate_canonical_baseline_or_candidate_bundles(tmp report = await execute_pipeline(request, evaluator=backend, optimizer=backend) - prompt_calls = [ - (call[1], call[2], call[3]["system_prompt"]) - for call in backend.calls - if call[0] == "evaluate" - ] + prompt_calls = [(call[1], call[2], call[3]["system_prompt"]) for call in backend.calls if call[0] == "evaluate"] assert prompt_calls == [ ("baseline", "train", "baseline\n"), ("baseline", "validation", "baseline\n"), @@ -243,20 +242,17 @@ async def test_backend_cannot_mutate_canonical_baseline_or_candidate_bundles(tmp assert optimize_call[1] == {"system_prompt": "baseline\n"} assert report.candidates[0]["candidate"].bundle() == {"system_prompt": "safe prompt"} assert report.candidates[0]["prompt_bundle"] == {"system_prompt": "safe prompt"} - assert report.audit["candidate_prompts"]["candidate_a"] == { - "system_prompt": "safe prompt" - } + assert report.audit["candidate_prompts"]["candidate_a"] == {"system_prompt": "safe prompt"} assert report.audit["candidate_prompt_hashes"]["candidate_a"] == { "system_prompt": hashlib.sha256(b"safe prompt").hexdigest() } candidate_artifact = report.audit["candidate_artifacts"]["candidate_a"] - artifact_prompt = ( - Path(request.output_dir) - / "runs" - / request.run_id - / "candidate_prompts" - / candidate_artifact - / "system_prompt.txt" + artifact_prompt = Path(request.output_dir).joinpath( + "runs", + request.run_id, + "candidate_prompts", + candidate_artifact, + "system_prompt.txt", ) assert artifact_prompt.read_text(encoding="utf-8") == "safe prompt" assert Path(request.target_prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "safe prompt" @@ -289,16 +285,15 @@ async def test_successful_backend_source_mutation_is_fatal( with pytest.raises(ConcurrentPromptUpdateError, match="source prompt.*changed"): await execute_pipeline(request, evaluator=backend, optimizer=backend) - assert Path(request.target_prompt_paths["system_prompt"]).read_text( - encoding="utf-8" - ) == "backend source mutation" + assert Path(request.target_prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "backend source mutation" assert not (Path(request.output_dir) / "optimization_report.json").exists() @pytest.mark.asyncio @pytest.mark.parametrize( "failure", - [RuntimeError("backend error after mutation"), asyncio.CancelledError()], + [RuntimeError("backend error after mutation"), + asyncio.CancelledError()], ids=["error", "cancellation"], ) async def test_backend_error_or_cancellation_with_source_mutation_is_integrity_fatal( @@ -314,16 +309,13 @@ async def test_backend_error_or_cancellation_with_source_mutation_is_integrity_f ) with pytest.raises( - ConcurrentPromptUpdateError, - match="source prompt.*changed", + ConcurrentPromptUpdateError, + match="source prompt.*changed", ) as error_info: await execute_pipeline(request, evaluator=backend, optimizer=backend) assert error_info.value.__cause__ is failure - assert not any( - call[0] == "evaluate" and call[1] == "candidate_b" - for call in backend.calls - ) + assert not any(call[0] == "evaluate" and call[1] == "candidate_b" for call in backend.calls) assert not (Path(request.output_dir) / "optimization_report.json").exists() @@ -342,10 +334,7 @@ async def test_clean_backend_cancellation_preserves_cancelled_error(tmp_path: Pa assert error_info.value is cancellation assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" - assert not any( - call[0] == "evaluate" and call[1] == "candidate_b" - for call in backend.calls - ) + assert not any(call[0] == "evaluate" and call[1] == "candidate_b" for call in backend.calls) @pytest.mark.asyncio @@ -385,9 +374,7 @@ async def test_candidate_evaluation_error_rejects_candidate_and_continues( report = await execute_pipeline(request, evaluator=backend, optimizer=backend) - candidate_calls = [ - (call[1], call[2]) for call in backend.calls if call[0] == "evaluate" and call[1] != "baseline" - ] + candidate_calls = [(call[1], call[2]) for call in backend.calls if call[0] == "evaluate" and call[1] != "baseline"] expected_a_calls = [("candidate_a", "train")] if failed_split == "validation": expected_a_calls.append(("candidate_a", "validation")) @@ -429,9 +416,7 @@ async def test_candidate_evaluation_error_rejects_candidate_and_continues( @pytest.mark.asyncio -async def test_candidate_evaluation_error_makes_later_budget_gate_fail_closed( - tmp_path: Path, -): +async def test_candidate_evaluation_error_makes_later_budget_gate_fail_closed(tmp_path: Path, ): request = _request( tmp_path, update_source=True, @@ -458,9 +443,7 @@ async def test_candidate_evaluation_error_makes_later_budget_gate_fail_closed( @pytest.mark.asyncio -async def test_all_candidate_evaluation_errors_finalize_rejected_audit_without_writeback( - tmp_path: Path, -): +async def test_all_candidate_evaluation_errors_finalize_rejected_audit_without_writeback(tmp_path: Path, ): request = _request(tmp_path, update_source=True) delegate = _safe_backend(candidate_count=2) backend = AllCandidatesFailingBackend( @@ -492,9 +475,7 @@ async def test_all_candidate_evaluation_errors_finalize_rejected_audit_without_w @pytest.mark.asyncio -async def test_candidate_evaluation_error_does_not_persist_backend_secret_text( - tmp_path: Path, -): +async def test_candidate_evaluation_error_does_not_persist_backend_secret_text(tmp_path: Path, ): request = _request(tmp_path) secret = "api_key=sk-private-value token=github-private-value" backend = FailingCandidateBackend( @@ -509,9 +490,7 @@ async def test_candidate_evaluation_error_does_not_persist_backend_secret_text( assert failure["message"] == "candidate evaluation failed; backend details withheld" assert failure["message_sha256"] == hashlib.sha256(secret.encode("utf-8")).hexdigest() serialized = report_module.report_to_json(report) + report_module.render_markdown(report) - serialized += (Path(request.output_dir) / "runs" / request.run_id / "audit.json").read_text( - encoding="utf-8" - ) + serialized += (Path(request.output_dir) / "runs" / request.run_id / "audit.json").read_text(encoding="utf-8") assert "sk-private-value" not in serialized assert "github-private-value" not in serialized @@ -560,6 +539,7 @@ async def test_candidate_failure_with_state_drift_aborts_run( request = _request(tmp_path, update_source=True) class StateDriftBackend(FailingCandidateBackend): + async def evaluate(self, **kwargs: Any) -> EvalResult: if (kwargs["prompt_id"], kwargs["split"]) == self.fail_on: if mutation_target == "prompt": @@ -588,7 +568,10 @@ async def test_rejected_candidate_never_changes_source_when_writeback_requested( request = _request( tmp_path, update_source=True, - gate_config={"min_val_score_improvement": 0.5, "max_total_cost": None}, + gate_config={ + "min_val_score_improvement": 0.5, + "max_total_cost": None + }, ) backend = _safe_backend(candidate_validation_score=0.0) before = (tmp_path / "prompt.txt").read_bytes() @@ -803,14 +786,14 @@ async def test_incomparable_case_results_are_rejected_without_delta_error( request = _request(tmp_path) for dataset_path in (Path(request.train_path), Path(request.validation_path)): dataset_path.write_text( - json.dumps( - { - "eval_cases": [ - {"eval_id": case_id, "session_input": {"state": {}}} - for case_id in dict.fromkeys(baseline_case_ids) - ] - } - ), + json.dumps({ + "eval_cases": [{ + "eval_id": case_id, + "session_input": { + "state": {} + } + } for case_id in dict.fromkeys(baseline_case_ids)] + }), encoding="utf-8", ) candidate = CandidatePrompt("candidate_a", "safe prompt", "safe", "diff") @@ -856,8 +839,12 @@ async def test_duplicate_candidate_ids_are_rejected(tmp_path: Path): @pytest.mark.parametrize( ("prompt_fields", "message"), [ - ({"other_prompt": "candidate"}, "bundle fields"), - ({"system_prompt": ""}, "non-empty string"), + ({ + "other_prompt": "candidate" + }, "bundle fields"), + ({ + "system_prompt": "" + }, "non-empty string"), ], ) async def test_candidate_bundle_must_match_target_snapshot( @@ -1002,9 +989,7 @@ def fail_outcome(report, paths): with pytest.raises(OSError, match="convenience artifact"): await execute_pipeline(request, evaluator=backend, optimizer=backend) - journal_path = ( - Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "writeback_journal.json" - ) + journal_path = (Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "writeback_journal.json") journal = json.loads(journal_path.read_text(encoding="utf-8")) assert (tmp_path / "prompt.txt").read_bytes() == b"safe prompt" assert journal["state"] == "applied" @@ -1023,12 +1008,7 @@ async def test_writeback_journal_is_committing_before_source_commit( real_commit = pipeline_module.commit_prompt_bundle def commit(snapshot, prompts): - journal_path = ( - Path(request.output_dir) - / "runs" - / f".{request.run_id}.tmp" - / "writeback_journal.json" - ) + journal_path = (Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "writeback_journal.json") journal = json.loads(journal_path.read_text(encoding="utf-8")) assert journal["state"] == "committing" return real_commit(snapshot, prompts) @@ -1076,7 +1056,12 @@ async def test_failed_writeback_is_persisted_in_final_report_and_journal( async def test_standard_evalset_protected_metadata_is_applied_to_gate(tmp_path: Path): request = _request(tmp_path) Path(request.train_path).write_text( - json.dumps({"eval_cases": [{"eval_id": "train", "session_input": {"state": {}}}]}), + json.dumps({"eval_cases": [{ + "eval_id": "train", + "session_input": { + "state": {} + } + }]}), encoding="utf-8", ) Path(request.validation_path).write_text( @@ -1124,19 +1109,24 @@ async def test_standard_evalset_protected_metadata_is_applied_to_gate(tmp_path: async def test_baseline_results_must_cover_every_snapshotted_dataset_case(tmp_path: Path): request = _request(tmp_path, update_source=True) Path(request.validation_path).write_text( - json.dumps( - { - "eval_cases": [ - {"eval_id": "validation_case", "session_input": {"state": {}}}, - { - "eval_id": "protected_missing", - "session_input": { - "state": {"eval_optimize_protected": True} - }, + json.dumps({ + "eval_cases": [ + { + "eval_id": "validation_case", + "session_input": { + "state": {} + } + }, + { + "eval_id": "protected_missing", + "session_input": { + "state": { + "eval_optimize_protected": True + } }, - ] - } - ), + }, + ] + }), encoding="utf-8", ) backend = _safe_backend(candidate_count=1) @@ -1160,9 +1150,7 @@ async def test_backend_neutral_core_rejects_same_resolved_train_and_validation_p @pytest.mark.asyncio -async def test_backend_neutral_core_rejects_hardlinked_train_and_validation( - tmp_path: Path, -): +async def test_backend_neutral_core_rejects_hardlinked_train_and_validation(tmp_path: Path, ): request = _request(tmp_path) validation_path = Path(request.validation_path) validation_path.unlink() @@ -1177,9 +1165,7 @@ async def test_backend_neutral_core_rejects_hardlinked_train_and_validation( assert not (Path(request.output_dir) / "runs").exists() -def test_factory_rejects_hardlinked_train_and_validation_before_backend_creation( - tmp_path: Path, -): +def test_factory_rejects_hardlinked_train_and_validation_before_backend_creation(tmp_path: Path, ): request = _request(tmp_path) validation_path = Path(request.validation_path) validation_path.unlink() @@ -1223,14 +1209,15 @@ async def test_backend_neutral_core_rejects_unsafe_or_colliding_target_fields( @pytest.mark.asyncio -async def test_backend_neutral_core_rejects_target_fields_for_same_resolved_file( - tmp_path: Path, -): +async def test_backend_neutral_core_rejects_target_fields_for_same_resolved_file(tmp_path: Path, ): request = _request(tmp_path) prompt_path = request.target_prompt_paths["system_prompt"] request = replace( request, - target_prompt_paths={"system_prompt": prompt_path, "router_prompt": prompt_path}, + target_prompt_paths={ + "system_prompt": prompt_path, + "router_prompt": prompt_path + }, ) backend = _safe_backend(candidate_count=1) @@ -1259,7 +1246,10 @@ async def test_backend_neutral_core_rejects_physical_or_casefold_target_aliases( second_path.write_text("second", encoding="utf-8") request = replace( request, - target_prompt_paths={"system_prompt": first_path, "router_prompt": second_path}, + target_prompt_paths={ + "system_prompt": first_path, + "router_prompt": second_path + }, ) backend = _safe_backend(candidate_count=1) @@ -1272,13 +1262,19 @@ async def test_backend_neutral_core_rejects_physical_or_casefold_target_aliases( def test_shared_path_validator_handles_missing_casefold_collisions(tmp_path: Path): pipeline_module.validate_distinct_file_paths( - {"first": tmp_path / "first.txt", "second": tmp_path / "second.txt"}, + { + "first": tmp_path / "first.txt", + "second": tmp_path / "second.txt" + }, context="test paths", ) with pytest.raises(ValueError, match="case-insensitively"): pipeline_module.validate_distinct_file_paths( - {"first": tmp_path / "Missing.txt", "second": tmp_path / "missing.txt"}, + { + "first": tmp_path / "Missing.txt", + "second": tmp_path / "missing.txt" + }, context="test paths", ) @@ -1297,7 +1293,12 @@ async def test_fake_gate_is_derived_from_immutable_optimizer_snapshot(tmp_path: base_request = _request(tmp_path) backend = _safe_backend(candidate_count=1) Path(base_request.optimizer_config_path).write_text( - json.dumps({"seed": 91, "gate": {"max_total_cost": None}}), + json.dumps({ + "seed": 91, + "gate": { + "max_total_cost": None + } + }), encoding="utf-8", ) request, selected_backend = build_pipeline_request_and_backend( @@ -1311,7 +1312,12 @@ async def test_fake_gate_is_derived_from_immutable_optimizer_snapshot(tmp_path: backend=backend, ) Path(base_request.optimizer_config_path).write_text( - json.dumps({"seed": 91, "gate": {"max_total_cost": 0.0}}), + json.dumps({ + "seed": 91, + "gate": { + "max_total_cost": 0.0 + } + }), encoding="utf-8", ) @@ -1337,7 +1343,12 @@ async def test_configured_protected_case_ids_must_exist_in_validation_metadata(t }, ) Path(request.validation_path).write_text( - json.dumps({"eval_cases": [{"eval_id": "present", "session_input": {"state": {}}}]}), + json.dumps({"eval_cases": [{ + "eval_id": "present", + "session_input": { + "state": {} + } + }]}), encoding="utf-8", ) backend = _safe_backend(candidate_count=1) @@ -1350,13 +1361,60 @@ async def test_configured_protected_case_ids_must_exist_in_validation_metadata(t @pytest.mark.parametrize( "payload", [ - {"eval_cases": [{"eval_id": 7, "session_input": {"state": {}}}]}, - {"eval_cases": [{"eval_id": "", "session_input": {"state": {}}}]}, - {"eval_cases": [{"eval_id": "case", "session_input": {"state": {"eval_optimize_protected": "yes"}}}]}, - {"eval_cases": [{"eval_id": "case", "session_input": {"state": {"eval_optimize_protected": None}}}]}, - {"cases": [{"case_id": 7, "protected": False}]}, - {"cases": [{"case_id": "case", "protected": "yes"}]}, - {"cases": [{"case_id": "case", "protected": 1}]}, + { + "eval_cases": [{ + "eval_id": 7, + "session_input": { + "state": {} + } + }] + }, + { + "eval_cases": [{ + "eval_id": "", + "session_input": { + "state": {} + } + }] + }, + { + "eval_cases": [{ + "eval_id": "case", + "session_input": { + "state": { + "eval_optimize_protected": "yes" + } + } + }] + }, + { + "eval_cases": [{ + "eval_id": "case", + "session_input": { + "state": { + "eval_optimize_protected": None + } + } + }] + }, + { + "cases": [{ + "case_id": 7, + "protected": False + }] + }, + { + "cases": [{ + "case_id": "case", + "protected": "yes" + }] + }, + { + "cases": [{ + "case_id": "case", + "protected": 1 + }] + }, ], ) async def test_validation_metadata_rejects_coerced_ids_and_protected_flags( @@ -1375,7 +1433,10 @@ async def test_validation_metadata_rejects_coerced_ids_and_protected_flags( async def test_validation_metadata_rejects_ambiguous_dual_schema(tmp_path: Path): request = _request(tmp_path) Path(request.validation_path).write_text( - json.dumps({"eval_cases": [], "cases": []}), + json.dumps({ + "eval_cases": [], + "cases": [] + }), encoding="utf-8", ) backend = _safe_backend(candidate_count=1) @@ -1404,9 +1465,7 @@ async def test_candidate_ids_are_artifact_safe_and_casefold_unique( async def test_candidate_prompts_must_be_valid_utf8_before_evaluation(tmp_path: Path): request = _request(tmp_path) backend = _safe_backend(candidate_count=0) - backend.candidates = [ - CandidatePrompt("candidate", "bad\ud800prompt", "safe", "diff") - ] + backend.candidates = [CandidatePrompt("candidate", "bad\ud800prompt", "safe", "diff")] with pytest.raises(ValueError, match="valid UTF-8"): await execute_pipeline(request, evaluator=backend, optimizer=backend) @@ -1416,7 +1475,9 @@ async def test_candidate_prompts_must_be_valid_utf8_before_evaluation(tmp_path: ("evaluate", "baseline", "validation"), ( "optimize", - {"system_prompt": "baseline\n"}, + { + "system_prompt": "baseline\n" + }, Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "optimizer", ), ] @@ -1511,6 +1572,7 @@ async def test_optimizer_raw_summary_must_be_json_compatible( class SnapshotMutationBackend(RecordingBackend): + def __init__(self, *, request: PipelineRequest, delegate: RecordingBackend) -> None: super().__init__( candidates=delegate.candidates, @@ -1551,6 +1613,7 @@ async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: class SnapshotTamperingBackend(RecordingBackend): + async def evaluate(self, **kwargs: Any) -> EvalResult: result = await super().evaluate(**kwargs) if kwargs["prompt_id"] == "baseline" and kwargs["split"] == "train": @@ -1598,14 +1661,11 @@ async def test_pipeline_uses_immutable_entry_snapshots_and_rejects_original_inpu assert used_paths.isdisjoint(original_paths) assert all(".snapshot-" in path.name for path in used_paths) assert all( - path.parent - in { + path.parent in { Path(request.train_path).resolve().parent, Path(request.validation_path).resolve().parent, Path(request.optimizer_config_path).resolve().parent, - } - for path in used_paths - ) + } for path in used_paths) assert report.writeback.status == "rejected" assert report.audit["writeback_journal"]["state"] == "conflict" assert "input changed" in (report.writeback.error or "") @@ -1621,7 +1681,9 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat "api_key": "plain-api-key", "nested": { "access_token": "plain-token", - "credentials": {"password": "plain-password"}, + "credentials": { + "password": "plain-password" + }, "github_token": "plain-github-token", "private_key": "plain-private-key", "cache_path": str(tmp_path / "private-cache"), @@ -1641,17 +1703,16 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat assert str(tmp_path.resolve()) not in payload assert report.run["reproducibility_shell"] == "powershell" assert "$EXTERNAL" in report.run["reproducibility_command"] - assert ( - report.audit["config_hash"] - == hashlib.sha256( - json.dumps(config_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") - ).hexdigest() - ) + assert (report.audit["config_hash"] == hashlib.sha256( + json.dumps(config_payload, ensure_ascii=False, sort_keys=True, + separators=(",", ":")).encode("utf-8")).hexdigest()) persisted = (Path(request.output_dir) / "optimization_report.json").read_text(encoding="utf-8") persisted_markdown = (Path(request.output_dir) / "optimization_report.md").read_text(encoding="utf-8") - config_snapshot = (Path(request.output_dir) / "runs" / request.run_id / "config.snapshot.json").read_text( - encoding="utf-8" - ) + config_snapshot = Path(request.output_dir).joinpath( + "runs", + request.run_id, + "config.snapshot.json", + ).read_text(encoding="utf-8") assert "plain-api-key" not in persisted + config_snapshot assert "plain-token" not in persisted + config_snapshot assert "plain-password" not in persisted + config_snapshot @@ -1659,12 +1720,10 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat assert "plain-private-key" not in persisted + config_snapshot assert "```powershell" in persisted_markdown assert str(tmp_path.resolve()) not in persisted_markdown - assert hashlib.sha256(config_snapshot.encode("utf-8")).hexdigest() == report.audit[ - "redacted_config_snapshot_sha256" - ] - assert report.audit["config_file_sha256"] == hashlib.sha256( - Path(request.optimizer_config_path).read_bytes() - ).hexdigest() + assert hashlib.sha256( + config_snapshot.encode("utf-8")).hexdigest() == report.audit["redacted_config_snapshot_sha256"] + assert report.audit["config_file_sha256"] == hashlib.sha256(Path( + request.optimizer_config_path).read_bytes()).hexdigest() def test_atomic_artifact_write_fsyncs_before_durable_replace( @@ -1810,6 +1869,7 @@ async def test_late_no_write_source_mutation_downgrades_journal_to_unknown( original_finalize = pipeline_module.finalize_run_artifacts def mutate_after_final_writes(report, paths, *, before_publish): + def mutate_then_verify(): Path(request.target_prompt_paths["system_prompt"]).write_text( "mutation after final artifacts", @@ -1824,17 +1884,10 @@ def mutate_then_verify(): with pytest.raises(ConcurrentPromptUpdateError, match="source prompt integrity changed"): await execute_pipeline(request, evaluator=backend, optimizer=backend) - journal_path = ( - Path(request.output_dir) - / "runs" - / f".{request.run_id}.tmp" - / "writeback_journal.json" - ) + journal_path = (Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "writeback_journal.json") journal = json.loads(journal_path.read_text(encoding="utf-8")) assert journal["state"] == "unknown" - assert journal["after_hashes"] == { - "system_prompt": hashlib.sha256(b"mutation after final artifacts").hexdigest() - } + assert journal["after_hashes"] == {"system_prompt": hashlib.sha256(b"mutation after final artifacts").hexdigest()} assert not (Path(request.output_dir) / "runs" / request.run_id).exists() @@ -1968,8 +2021,7 @@ def _request( optimizer_config_path=config_path, output_dir=tmp_path / "out", target_prompt_paths={"system_prompt": prompt_path}, - gate_config=gate_config - or { + gate_config=gate_config or { "min_val_score_improvement": 0.0, "allow_new_hard_fail": False, "protected_case_ids": [], @@ -1998,7 +2050,8 @@ def _safe_backend( CandidatePrompt("candidate_b", "other prompt", "other", "diff b"), ][:candidate_count] results = { - ("baseline", "train"): _eval( + ("baseline", "train"): + _eval( "baseline", "train", ["train_case"], @@ -2006,7 +2059,8 @@ def _safe_backend( cost=baseline_train_cost, failure_category="train_only", ), - ("baseline", "validation"): _eval( + ("baseline", "validation"): + _eval( "baseline", "validation", ["validation_case"], @@ -2055,8 +2109,7 @@ def _eval( output="output", hard_failed=score <= 0.0, failure_category=failure_category if score < 1.0 else None, - ) - for case_id in case_ids + ) for case_id in case_ids ] return EvalResult( prompt_id=prompt_id, @@ -2081,8 +2134,7 @@ def _eval_scores( passed=score >= 1.0, output="output", hard_failed=score <= 0.0, - ) - for case_id, score in scores + ) for case_id, score in scores ] return EvalResult( prompt_id=prompt_id, diff --git a/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py b/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py index 937f3b43..b8d45aca 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py +++ b/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py @@ -16,16 +16,15 @@ from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationRound from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import ( - FailingCandidateBackend, -) + FailingCandidateBackend, ) from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import ( - RecordingBackend, -) + RecordingBackend, ) from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import _request from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import _safe_backend class _BaselineCrashBackend(RecordingBackend): + async def evaluate(self, **kwargs: Any): artifact_dir = Path(kwargs["artifact_dir"]) artifact_dir.joinpath("backend-started.txt").write_text("started\n", encoding="utf-8") @@ -97,6 +96,7 @@ async def _source_report(tmp_path: Path, label: str): class _FakeRenameAt2: + def __init__(self, result: int) -> None: self.result = result self.calls: list[tuple[Any, ...]] = [] @@ -121,6 +121,7 @@ class _FakeLibc: renameat2 = function class _CtypesProxy: + def CDLL(self, name: Any, *, use_errno: bool): assert name is None assert use_errno is True @@ -185,10 +186,12 @@ def test_posix_rename_noreplace_fails_closed_when_unsupported( ): real_ctypes = report_module.ctypes if failure == "missing_symbol": + class _LibcWithoutRenameAt2: pass class _MissingSymbolProxy: + def CDLL(self, name: Any, *, use_errno: bool): return _LibcWithoutRenameAt2() @@ -290,10 +293,10 @@ async def test_pipeline_rejects_duplicate_round_ids_before_report_artifact_write assert temp_run.is_dir() assert not _final_run_dir(request).exists() for report_artifact in ( - "pre_write_report.json", - "artifact_manifest.json", - "rounds.json", - "rounds", + "pre_write_report.json", + "artifact_manifest.json", + "rounds.json", + "rounds", ): assert not (temp_run / report_artifact).exists() @@ -315,19 +318,19 @@ async def test_direct_report_writers_reject_duplicate_round_ids_before_writing( run_id = f"duplicate_rounds_{writer}" report = replace( source_report, - run={**source_report.run, "run_id": run_id}, + run={ + **source_report.run, "run_id": run_id + }, rounds=[_round(7, rationale="first"), _round(7, rationale="duplicate")], ) output_dir = tmp_path / f"direct-{writer}" - if writer == "prepare": - paths = report_module.reserve_run_artifacts(output_dir, run_id=run_id) - operation = lambda: report_module.prepare_run_artifacts(report, paths) - else: - operation = lambda: report_module.write_reports(report, output_dir) - with pytest.raises(ValueError, match=r"duplicate round_id.*7"): - operation() + if writer == "prepare": + paths = report_module.reserve_run_artifacts(output_dir, run_id=run_id) + report_module.prepare_run_artifacts(report, paths) + else: + report_module.write_reports(report, output_dir) temp_run = output_dir / "runs" / f".{run_id}.tmp" assert temp_run.is_dir() @@ -393,9 +396,7 @@ def fail_final_markdown(path: Path, content: str) -> None: @pytest.mark.asyncio -async def test_before_publish_failure_refreshes_manifest_without_replacing_original_error( - tmp_path: Path, -): +async def test_before_publish_failure_refreshes_manifest_without_replacing_original_error(tmp_path: Path, ): source_root = tmp_path / "callback-source" source_root.mkdir() source_request = _request(source_root) @@ -412,8 +413,12 @@ async def test_before_publish_failure_refreshes_manifest_without_replacing_origi } report = replace( source_report, - run={**source_report.run, "run_id": run_id}, - audit={**source_report.audit, "writeback_journal": journal}, + run={ + **source_report.run, "run_id": run_id + }, + audit={ + **source_report.audit, "writeback_journal": journal + }, ) paths = report_module.reserve_run_artifacts(tmp_path / "callback-output", run_id=run_id) report_module.prepare_run_artifacts(report, paths) @@ -428,7 +433,9 @@ def mutate_journal_then_fail() -> None: **journal, "state": "unknown", "error": "source prompt integrity changed before publication", - "after_hashes": {"system_prompt": "changed"}, + "after_hashes": { + "system_prompt": "changed" + }, }, ) raise _PromptIntegritySentinel("prompt drift sentinel") @@ -443,22 +450,16 @@ def mutate_journal_then_fail() -> None: assert type(caught.value) is _PromptIntegritySentinel assert paths.temp_run_dir.is_dir() assert not paths.final_run_dir.exists() - manifest = json.loads( - (paths.temp_run_dir / "artifact_manifest.json").read_text(encoding="utf-8") - ) + manifest = json.loads((paths.temp_run_dir / "artifact_manifest.json").read_text(encoding="utf-8")) _assert_manifest_records_match_files(paths.temp_run_dir, manifest) - journal_record = next( - record for record in manifest["files"] if record["path"] == "writeback_journal.json" - ) + journal_record = next(record for record in manifest["files"] if record["path"] == "writeback_journal.json") journal_bytes = (paths.temp_run_dir / "writeback_journal.json").read_bytes() assert journal_record["sha256"] == hashlib.sha256(journal_bytes).hexdigest() assert journal_record["size_bytes"] == len(journal_bytes) @pytest.mark.asyncio -async def test_prepare_rejects_linked_top_level_audit_directory_before_outside_write( - tmp_path: Path, -): +async def test_prepare_rejects_linked_top_level_audit_directory_before_outside_write(tmp_path: Path, ): source_report = await _source_report(tmp_path, "linked-top-source") run_id = "linked_top_level" report = replace(source_report, run={**source_report.run, "run_id": run_id}) @@ -601,9 +602,7 @@ async def test_fresh_published_run_has_positive_duration_and_only_strict_json(tm payload = artifact.read_text(encoding="utf-8") json.loads( payload, - parse_constant=lambda value: (_ for _ in ()).throw( - ValueError(f"non-standard JSON constant: {value}") - ), + parse_constant=lambda value: (_ for _ in ()).throw(ValueError(f"non-standard JSON constant: {value}")), ) @@ -649,10 +648,10 @@ async def test_manifest_covers_complete_audit_set_and_partial_candidate_results( declared_files = {item["path"] for item in manifest["files"]} for group in ( - artifacts["reports"], - artifacts["baseline_case_results"], - artifacts["baseline_prompts"], - artifacts["audit_records"], + artifacts["reports"], + artifacts["baseline_case_results"], + artifacts["baseline_prompts"], + artifacts["audit_records"], ): assert set(group.values()) <= declared_files assert set(artifacts["rounds"]) <= declared_files @@ -660,9 +659,8 @@ async def test_manifest_covers_complete_audit_set_and_partial_candidate_results( assert candidate["diff"] in declared_files assert set(candidate["prompt_bundle"].values()) <= declared_files assert set(candidate["case_results"].values()) <= declared_files - assert json.loads((run_dir / artifacts["audit_records"]["evaluation_failures"]).read_text(encoding="utf-8"))[ - "candidate_a" - ]["stage"] == "validation" + assert json.loads((run_dir / artifacts["audit_records"]["evaluation_failures"]).read_text( + encoding="utf-8"))["candidate_a"]["stage"] == "validation" @pytest.mark.asyncio @@ -678,13 +676,11 @@ async def test_convenience_reports_are_exact_copies_created_after_run_publicatio def observe_copy(path: Path, content: bytes) -> None: path = Path(path) if path.parent == Path(request.output_dir) and path.name.startswith("optimization_report"): - copy_events.append( - ( - path.name, - _final_run_dir(request).is_dir(), - _temp_run_dir(request).exists(), - ) - ) + copy_events.append(( + path.name, + _final_run_dir(request).is_dir(), + _temp_run_dir(request).exists(), + )) original_write_bytes(path, content) monkeypatch.setattr(report_module, "_atomic_write_bytes", observe_copy) @@ -720,3 +716,32 @@ async def test_runtime_input_hashes_match_exact_entry_bytes(tmp_path: Path): for role, content in exact_inputs.items(): assert hashes[role] == hashlib.sha256(content).hexdigest() assert hashes["target_prompts"]["system_prompt"] == hashlib.sha256(prompt_bytes).hexdigest() + + +def test_committed_example_matches_committed_inputs_without_personal_paths(): + example_root = Path(__file__).resolve().parents[1] + report_path = example_root / "outputs" / "optimization_report.example.json" + payload = json.loads(report_path.read_text(encoding="utf-8")) + input_hashes = payload["audit"]["input_hashes"] + expected_inputs = { + "train": example_root / "data" / "train.evalset.json", + "validation": example_root / "data" / "val.evalset.json", + "optimizer": example_root / "data" / "optimizer.json", + } + + for role, path in expected_inputs.items(): + assert input_hashes[role] == hashlib.sha256(path.read_bytes()).hexdigest() + prompt_path = example_root / "prompts" / "baseline_system_prompt.txt" + assert input_hashes["target_prompts"]["system_prompt"] == hashlib.sha256(prompt_path.read_bytes()).hexdigest() + + assert payload["schema_version"] == "eval_optimize_loop.v2" + assert payload["run"]["run_id"] == "example" + assert payload["audit"]["duration_seconds"] > 0 + assert payload["rounds"] + assert all(round_record["duration_seconds"] > 0 for round_record in payload["rounds"]) + + serialized = json.dumps(payload, ensure_ascii=False) + assert "C:\\Users\\" not in serialized + assert "/Users/" not in serialized + assert "/home/" not in serialized + assert "partial_applied" not in serialized diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py index 88cb8df7..c8e7cc83 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -41,7 +41,8 @@ async def test_fake_backend_implements_unified_contract_with_real_trace(tmp_path dataset_path = tmp_path / "fake_train.evalset.json" dataset_path.write_text( json.dumps({ - "split": "train", + "split": + "train", "cases": [{ "case_id": "case_a", "input": "Mention latency and retries.", @@ -113,7 +114,9 @@ async def test_fake_backend_wraps_candidates_in_complete_optimization_result(tmp result = await backend_module.FakeBackend(seed=91).optimize_candidates( baseline_prompts={"system_prompt": baseline_prompt}, baseline_train=baseline_train, - failure_summary={"by_category": {"format_violation": 1}}, + failure_summary={"by_category": { + "format_violation": 1 + }}, train_path=DEFAULT_TRAIN, validation_path=DEFAULT_VAL, config_path=DEFAULT_OPTIMIZER_CONFIG, @@ -147,7 +150,9 @@ async def test_fake_backend_distributes_measured_batch_proposal_duration( result = await backend_module.FakeBackend(seed=91).optimize_candidates( baseline_prompts={"system_prompt": "baseline prompt\n"}, baseline_train=_failed_fake_train_result(), - failure_summary={"by_category": {"format_violation": 1}}, + failure_summary={"by_category": { + "format_violation": 1 + }}, train_path=tmp_path / "train.evalset.json", validation_path=tmp_path / "validation.evalset.json", config_path=tmp_path / "optimizer.json", @@ -159,9 +164,7 @@ async def test_fake_backend_distributes_measured_batch_proposal_duration( assert all(duration > 0 and math.isfinite(duration) for duration in durations) assert sum(durations) == pytest.approx(0.006) assert result.raw_summary["proposal_duration_seconds"] == pytest.approx(0.006) - assert result.raw_summary["round_duration_allocation"] == ( - "equal_share_of_batch_proposal_duration" - ) + assert result.raw_summary["round_duration_allocation"] == ("equal_share_of_batch_proposal_duration") @pytest.mark.asyncio @@ -183,7 +186,9 @@ async def test_fake_backend_uses_clock_resolution_when_batch_timer_does_not_adva result = await backend_module.FakeBackend(seed=91).optimize_candidates( baseline_prompts={"system_prompt": "baseline prompt\n"}, baseline_train=_failed_fake_train_result(), - failure_summary={"by_category": {"format_violation": 1}}, + failure_summary={"by_category": { + "format_violation": 1 + }}, train_path=tmp_path / "train.evalset.json", validation_path=tmp_path / "validation.evalset.json", config_path=tmp_path / "optimizer.json", @@ -261,7 +266,10 @@ async def test_sdk_backend_maps_rounds_deduplicates_bundles_and_marks_cost_incom backend = SDKBackend( prompt_path=system_path, call_agent_path="fake_call_agent_module:call_agent", - target_prompt_paths={"system_prompt": system_path, "router_prompt": router_path}, + target_prompt_paths={ + "system_prompt": system_path, + "router_prompt": router_path + }, ) result = await backend.optimize_candidates( @@ -526,9 +534,19 @@ def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): assert case.output == "last output" assert case.trace_available is True assert case.trace == { - "user_content": {"parts": [{"text": "last question"}]}, - "final_response": {"parts": [{"text": "last output"}]}, - "intermediate_data": {"step": 2}, + "user_content": { + "parts": [{ + "text": "last question" + }] + }, + "final_response": { + "parts": [{ + "text": "last output" + }] + }, + "intermediate_data": { + "step": 2 + }, } assert case.expected_failure_category == "format_violation" assert case.failure_category == "final_response_mismatch" @@ -544,13 +562,9 @@ def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): ], ) def test_sdk_result_mapping_rejects_case_id_set_mismatch(sdk_case_ids, expected_case_ids, message): - sdk_runs = { - case_id: [_sdk_case_run(case_id, status="PASSED", metrics=[])] - for case_id in sdk_case_ids - } + sdk_runs = {case_id: [_sdk_case_run(case_id, status="PASSED", metrics=[])] for case_id in sdk_case_ids} expected_cases = [ - EvalCase(case_id=case_id, split="validation", input="", expectation={}) - for case_id in expected_case_ids + EvalCase(case_id=case_id, split="validation", input="", expectation={}) for case_id in expected_case_ids ] with pytest.raises(ValueError, match=message): @@ -568,8 +582,7 @@ def test_sdk_result_mapping_rejects_duplicate_case_ids_across_eval_sets(): results_by_eval_set_id={ "set_a": types.SimpleNamespace(eval_results_by_eval_id={"case_a": [run]}), "set_b": types.SimpleNamespace(eval_results_by_eval_id={"case_a": [run]}), - } - ) + }) expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] with pytest.raises(ValueError, match="duplicate.*case_a"): @@ -681,18 +694,20 @@ def test_sdk_expected_cases_parse_standard_evalset_metadata(tmp_path: Path): "invocation_id": "case-1-turn-0", "user_content": { "role": "user", - "parts": [{"text": "earlier query"}], + "parts": [{ + "text": "earlier query" + }], }, "final_response": { "role": "model", - "parts": [{"text": "earlier expected"}], + "parts": [{ + "text": "earlier expected" + }], }, }, ) dataset_path.write_text( - json.dumps( - _sdk_evalset_payload([case_payload]) - ), + json.dumps(_sdk_evalset_payload([case_payload])), encoding="utf-8", ) @@ -718,12 +733,10 @@ def test_sdk_expected_cases_parse_standard_evalset_metadata(tmp_path: Path): def test_sdk_expected_cases_reject_duplicate_eval_ids(tmp_path: Path): dataset_path = tmp_path / "duplicate.evalset.json" dataset_path.write_text( - json.dumps( - _sdk_evalset_payload([ - _sdk_eval_case_payload("case-1"), - _sdk_eval_case_payload("case-1"), - ]) - ), + json.dumps(_sdk_evalset_payload([ + _sdk_eval_case_payload("case-1"), + _sdk_eval_case_payload("case-1"), + ])), encoding="utf-8", ) @@ -755,7 +768,10 @@ def test_sdk_expected_cases_require_expectation_metadata(tmp_path: Path): def test_sdk_expected_cases_reject_invalid_eval_cases_shape(tmp_path: Path): dataset_path = tmp_path / "invalid_shape.evalset.json" dataset_path.write_text( - json.dumps({"eval_set_id": "set", "eval_cases": {}}), + json.dumps({ + "eval_set_id": "set", + "eval_cases": {} + }), encoding="utf-8", ) @@ -781,8 +797,7 @@ async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_byt expected="answer", expected_failure_category="format_violation", ) - ]) - ), + ])), encoding="utf-8", ) from trpc_agent_sdk.evaluation import EvalSet @@ -826,13 +841,9 @@ async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_byt assert calls["observed_candidate"] is True assert calls["eval_result_output_dir"] == str(tmp_path / "sdk_eval") eval_config_path = Path(calls["eval_metrics_file_path_or_dir"]) - eval_config = EvalConfig.model_validate_json( - eval_config_path.read_text(encoding="utf-8") - ) + eval_config = EvalConfig.model_validate_json(eval_config_path.read_text(encoding="utf-8")) assert eval_config.criteria == {"final_response_avg_score": 1.0} - assert [metric.metric_name for metric in eval_config.get_eval_metrics()] == [ - "final_response_avg_score" - ] + assert [metric.metric_name for metric in eval_config.get_eval_metrics()] == ["final_response_avg_score"] assert prompt_path.read_bytes() == original_bytes assert mapped.cases[0].trace_available is True assert mapped.cases[0].expected_failure_category == "format_violation" @@ -845,15 +856,11 @@ async def test_sdk_backend_evaluate_propagates_non_case_failure_even_with_result ): dataset_path = tmp_path / "validation.evalset.json" dataset_path.write_text( - json.dumps( - _sdk_evalset_payload([ - _sdk_eval_case_payload( - "case_a", - query="question", - expected="answer", - ) - ]) - ), + json.dumps(_sdk_evalset_payload([_sdk_eval_case_payload( + "case_a", + query="question", + expected="answer", + )])), encoding="utf-8", ) prompt_path = tmp_path / "prompt.txt" @@ -900,15 +907,11 @@ async def test_sdk_backend_evaluate_real_agent_evaluator_smoke( ): dataset_path = (tmp_path / "validation.evalset.json").resolve() dataset_path.write_text( - json.dumps( - _sdk_evalset_payload([ - _sdk_eval_case_payload( - "case_a", - query="question", - expected="answer", - ) - ]) - ), + json.dumps(_sdk_evalset_payload([_sdk_eval_case_payload( + "case_a", + query="question", + expected="answer", + )])), encoding="utf-8", ) prompt_path = tmp_path / "prompt.txt" @@ -1012,7 +1015,9 @@ def test_sdk_backend_router_prompt_only_can_succeed(tmp_path: Path, monkeypatch) candidates = SDKBackend( prompt_path=tmp_path / "unused_system.txt", call_agent_path="fake_call_agent_module:call_agent", - target_prompt_paths={"router_prompt": router_path}, + target_prompt_paths={ + "router_prompt": router_path + }, ).optimize( baseline_prompt="unused", train_path=tmp_path / "train.evalset.json", @@ -1033,7 +1038,9 @@ def test_sdk_backend_skill_prompt_only_can_succeed(tmp_path: Path, monkeypatch): candidates = SDKBackend( prompt_path=tmp_path / "unused_system.txt", call_agent_path="fake_call_agent_module:call_agent", - target_prompt_paths={"skill_prompt": skill_path}, + target_prompt_paths={ + "skill_prompt": skill_path + }, ).optimize( baseline_prompt="unused", train_path=tmp_path / "train.evalset.json", @@ -1055,7 +1062,10 @@ def test_sdk_backend_missing_registered_best_prompt_field_is_clear(tmp_path: Pat backend = SDKBackend( prompt_path=tmp_path / "unused_system.txt", call_agent_path="fake_call_agent_module:call_agent", - target_prompt_paths={"router_prompt": router_path, "skill_prompt": skill_path}, + target_prompt_paths={ + "router_prompt": router_path, + "skill_prompt": skill_path + }, ) with pytest.raises(ValueError, match="best_prompts.*missing registered target fields.*skill_prompt"): @@ -1323,9 +1333,8 @@ def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, mo candidate_artifact = report.audit["candidate_artifacts"]["sdk_best"] assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / f"{candidate_artifact}.diff").is_file() assert calls["update_source"] is False - assert calls["output_dir"].endswith("runs\\.sdk_test_run.tmp\\optimizer") or calls[ - "output_dir" - ].endswith("runs/.sdk_test_run.tmp/optimizer") + assert calls["output_dir"].endswith("runs\\.sdk_test_run.tmp\\optimizer") or calls["output_dir"].endswith( + "runs/.sdk_test_run.tmp/optimizer") assert calls["evaluation_count"] == 4 command = report.run["reproducibility_command"] assert "--sdk-call-agent fake_call_agent_module:call_agent" in command @@ -1350,7 +1359,14 @@ def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp encoding="utf-8", ) optimizer_path.write_text( - json.dumps({"seed": "sdk-owned-seed", "optimize": {"algorithm": {"name": "gepa_reflective"}}}), + json.dumps({ + "seed": "sdk-owned-seed", + "optimize": { + "algorithm": { + "name": "gepa_reflective" + } + } + }), encoding="utf-8", ) prompt_path.write_text("baseline", encoding="utf-8") @@ -1820,15 +1836,21 @@ def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: } run_dir = tmp_path / "sdk_run" / "runs" / "sdk_multi_target" candidate_artifact = report.audit["candidate_artifacts"]["sdk_best"] - assert (run_dir / "candidate_prompts" / candidate_artifact / "system_prompt.txt").read_text( - encoding="utf-8" - ) == "optimized system" - assert (run_dir / "candidate_prompts" / candidate_artifact / "router_prompt.txt").read_text( - encoding="utf-8" - ) == "optimized router" - assert (run_dir / "candidate_prompts" / candidate_artifact / "skill_prompt.txt").read_text( - encoding="utf-8" - ) == "optimized skill" + assert run_dir.joinpath( + "candidate_prompts", + candidate_artifact, + "system_prompt.txt", + ).read_text(encoding="utf-8") == "optimized system" + assert run_dir.joinpath( + "candidate_prompts", + candidate_artifact, + "router_prompt.txt", + ).read_text(encoding="utf-8") == "optimized router" + assert run_dir.joinpath( + "candidate_prompts", + candidate_artifact, + "skill_prompt.txt", + ).read_text(encoding="utf-8") == "optimized skill" input_hashes = json.loads((run_dir / "input_hashes.json").read_text(encoding="utf-8")) assert set(input_hashes["target_prompts"]) == { "system_prompt", @@ -1905,6 +1927,7 @@ def _install_fake_sdk( calls = {} class FakeTargetPrompt: + def __init__(self): self.paths = [] @@ -1913,6 +1936,7 @@ def add_path(self, name, path): return self class FakeAgentOptimizer: + @staticmethod async def optimize(**kwargs): calls.update(kwargs) @@ -1949,7 +1973,11 @@ async def optimize(**kwargs): best_metric_breakdown={"exact_match": best_pass_rate}, metric_thresholds={"exact_match": 0.7}, total_llm_cost=total_llm_cost, - total_token_usage={"prompt": 100, "completion": 25, "total": 125}, + total_token_usage={ + "prompt": 100, + "completion": 25, + "total": 125 + }, duration_seconds=duration_seconds, started_at=started_at, total_rounds=len(effective_rounds), @@ -1957,6 +1985,7 @@ async def optimize(**kwargs): ) class FakeEvalConfig: + def __init__(self, **kwargs): self.payload = kwargs @@ -1967,6 +1996,7 @@ class FakeEvaluationCasesFailed(Exception): pass class FakeAgentEvaluator: + @staticmethod def get_executer(dataset_path, **kwargs): evaluation_index = int(calls.get("evaluation_count", 0)) @@ -1995,6 +2025,7 @@ def get_executer(dataset_path, **kwargs): }) class FakeExecuter: + async def evaluate(self): return None @@ -2078,7 +2109,11 @@ def _sdk_round( per_field_diagnosis={}, reflection_lm_calls=1, round_llm_cost=round_llm_cost, - round_token_usage={"prompt": 0, "completion": 0, "total": 0}, + round_token_usage={ + "prompt": 0, + "completion": 0, + "total": 0 + }, started_at="2026-07-04T12:00:00+00:00", duration_seconds=duration_seconds, kind="reflective", @@ -2117,8 +2152,12 @@ def _sdk_case_run( if output is not None or user_content is not None or intermediate_data is not None: actual_invocation = types.SimpleNamespace( invocation_id=f"{case_id}_invocation", - user_content={"parts": [{"text": user_content or ""}]}, - final_response={"parts": [{"text": output or ""}]} if output is not None else None, + user_content={"parts": [{ + "text": user_content or "" + }]}, + final_response={"parts": [{ + "text": output or "" + }]} if output is not None else None, intermediate_data=intermediate_data, creation_timestamp=0.0, ) @@ -2127,8 +2166,7 @@ def _sdk_case_run( actual_invocation=actual_invocation, expected_invocation=None, eval_metric_results=list(metrics), - ) - ) + )) return types.SimpleNamespace( eval_set_id="set_a", eval_id=case_id, @@ -2146,12 +2184,12 @@ def _sdk_case_run( def _sdk_evaluate_result(runs_by_case_id: dict[str, list[object]]): return types.SimpleNamespace( results_by_eval_set_id={ - "set_a": types.SimpleNamespace( + "set_a": + types.SimpleNamespace( eval_results_by_eval_id=dict(runs_by_case_id), num_runs=max((len(runs) for runs in runs_by_case_id.values()), default=1), ) - } - ) + }) def _sdk_evalset_payload(eval_cases: list[dict[str, object]]) -> dict[str, object]: @@ -2171,16 +2209,21 @@ def _sdk_eval_case_payload( protected: bool = False, ) -> dict[str, object]: return { - "eval_id": eval_id, + "eval_id": + eval_id, "conversation": [{ "invocation_id": f"{eval_id}-turn-1", "user_content": { "role": "user", - "parts": [{"text": query}], + "parts": [{ + "text": query + }], }, "final_response": { "role": "model", - "parts": [{"text": expected}], + "parts": [{ + "text": expected + }], }, }], "session_input": { @@ -2209,6 +2252,7 @@ def _install_fake_agent_evaluator( calls: dict[str, object] = {} class FakeExecuter: + async def evaluate(self): calls["observed_candidate"] = bool(on_evaluate()) if evaluation_error is not None: @@ -2218,6 +2262,7 @@ def get_result(self): return result class FakeAgentEvaluator: + @staticmethod def get_executer(dataset_path, **kwargs): calls["dataset_path"] = dataset_path @@ -2247,8 +2292,14 @@ def _write_sdk_optimizer_config(tmp_path: Path) -> Path: path = tmp_path / "sdk_optimizer.json" path.write_text( json.dumps({ - "evaluate": {"metrics": []}, - "optimize": {"algorithm": {"name": "gepa_reflective"}}, + "evaluate": { + "metrics": [] + }, + "optimize": { + "algorithm": { + "name": "gepa_reflective" + } + }, }), encoding="utf-8", ) @@ -2263,12 +2314,11 @@ def _write_gate_config( ) -> Path: path = tmp_path / "wrapper_gate.json" path.write_text( - json.dumps({ - "gate": { + json.dumps( + {"gate": { "min_val_score_improvement": min_val_score_improvement, "max_total_cost": max_total_cost, - } - }), + }}), encoding="utf-8", ) return path diff --git a/examples/optimization/eval_optimize_loop/tests/test_writeback.py b/examples/optimization/eval_optimize_loop/tests/test_writeback.py index dad140c0..b325334c 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_writeback.py +++ b/examples/optimization/eval_optimize_loop/tests/test_writeback.py @@ -8,18 +8,14 @@ from examples.optimization.eval_optimize_loop.eval_loop import writeback from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( - ConcurrentPromptUpdateError, -) + ConcurrentPromptUpdateError, ) from examples.optimization.eval_optimize_loop.eval_loop.writeback import PromptRestorationError from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( - commit_prompt_bundle, -) + commit_prompt_bundle, ) from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( - snapshot_prompt_files, -) + snapshot_prompt_files, ) from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( - temporary_prompt_bundle, -) + temporary_prompt_bundle, ) def _prompt_files(tmp_path: Path) -> tuple[dict[str, Path], dict[str, bytes]]: @@ -66,12 +62,13 @@ def test_snapshot_prompt_files_rejects_hardlink_aliases(tmp_path: Path): def test_temporary_prompt_bundle_restores_exact_bytes_after_body_error(tmp_path: Path): paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) + candidate = { + "system": "candidate system", + "user": "candidate \u7528\u6237", + } with pytest.raises(RuntimeError, match="candidate failed"): - with temporary_prompt_bundle( - snapshot, - {"system": "candidate system", "user": "candidate \u7528\u6237"}, - ): + with temporary_prompt_bundle(snapshot, candidate): assert paths["system"].read_bytes() == b"candidate system" assert paths["user"].read_bytes() == "candidate \u7528\u6237".encode() raise RuntimeError("candidate failed") @@ -94,6 +91,10 @@ def test_temporary_rejects_stale_snapshot_without_overwriting_external_update(tm paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) external_content = b"external before temporary entry" + candidate = { + "system": "candidate system", + "user": "candidate user", + } paths["user"].write_bytes(external_content) original_replace = os.replace replace_calls = 0 @@ -106,10 +107,7 @@ def count_replace(source: str | bytes | Path, destination: str | bytes | Path): monkeypatch.setattr(writeback.os, "replace", count_replace) with pytest.raises(ConcurrentPromptUpdateError, match="changed"): - with temporary_prompt_bundle( - snapshot, - {"system": "candidate system", "user": "candidate user"}, - ): + with temporary_prompt_bundle(snapshot, candidate): pass assert replace_calls == 0 @@ -121,12 +119,13 @@ def test_temporary_restore_preserves_external_update_and_reports_conflict(tmp_pa paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) external_content = b"external during temporary evaluation" + candidate = { + "system": "candidate system", + "user": "candidate user", + } with pytest.raises(PromptRestorationError, match="restore"): - with temporary_prompt_bundle( - snapshot, - {"system": "candidate system", "user": "candidate user"}, - ): + with temporary_prompt_bundle(snapshot, candidate): paths["user"].write_bytes(external_content) assert paths["system"].read_bytes() == baseline["system"] @@ -137,12 +136,13 @@ def test_temporary_body_error_remains_primary_when_restore_conflicts(tmp_path: P paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) external_content = b"external before failed body exits" + candidate = { + "system": "candidate system", + "user": "candidate user", + } with pytest.raises(RuntimeError, match="candidate failed") as error_info: - with temporary_prompt_bundle( - snapshot, - {"system": "candidate system", "user": "candidate user"}, - ): + with temporary_prompt_bundle(snapshot, candidate): paths["user"].write_bytes(external_content) raise RuntimeError("candidate failed") @@ -163,6 +163,10 @@ def test_temporary_partial_install_restores_written_file_and_preserves_external_ paths, baseline = _prompt_files(tmp_path) snapshot = snapshot_prompt_files(paths) external_content = b"external before second temporary replace" + candidate = { + "system": "candidate system", + "user": "candidate user", + } original_fsync = os.fsync fsync_calls = 0 @@ -176,10 +180,7 @@ def inject_external_before_second_precondition(file_descriptor: int): monkeypatch.setattr(writeback.os, "fsync", inject_external_before_second_precondition) with pytest.raises(ConcurrentPromptUpdateError, match="changed"): - with temporary_prompt_bundle( - snapshot, - {"system": "candidate system", "user": "candidate user"}, - ): + with temporary_prompt_bundle(snapshot, candidate): pass assert paths["system"].read_bytes() == baseline["system"] @@ -193,6 +194,10 @@ def test_temporary_verifies_complete_candidate_bundle_before_entering_body(tmp_p original_replace = os.replace replace_calls = 0 body_entered = False + candidate = { + "system": "candidate system", + "user": "candidate user", + } def update_first_file_after_second_replace(source: str | bytes | Path, destination: str | bytes | Path): nonlocal replace_calls @@ -204,10 +209,7 @@ def update_first_file_after_second_replace(source: str | bytes | Path, destinati monkeypatch.setattr(writeback.os, "replace", update_first_file_after_second_replace) with pytest.raises((ConcurrentPromptUpdateError, RuntimeError)): - with temporary_prompt_bundle( - snapshot, - {"system": "candidate system", "user": "candidate user"}, - ): + with temporary_prompt_bundle(snapshot, candidate): body_entered = True assert not body_entered @@ -229,7 +231,10 @@ def unexpected_write(path: Path, content: bytes) -> None: with pytest.raises(ConcurrentPromptUpdateError, match="changed"): commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert paths["system"].read_bytes() == baseline["system"] @@ -255,7 +260,10 @@ def inject_external_before_precondition(file_descriptor: int): with pytest.raises(ConcurrentPromptUpdateError, match="changed"): commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert external_injected @@ -281,7 +289,10 @@ def inject_external_before_second_precondition(file_descriptor: int): result = commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert result.status == "rollback_failed" @@ -309,7 +320,10 @@ def update_after_second_replace(source: str | bytes | Path, destination: str | b result = commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert result.status == "rollback_failed" @@ -348,7 +362,10 @@ def fail_second_replace(source: str | bytes | Path, destination: str | bytes | P result = commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert result.status == "rollback_failed" @@ -405,7 +422,10 @@ def fail_once_for_candidate_system(path: Path): result = commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert hash_read_failed @@ -485,7 +505,10 @@ def fail_second_replace(source: str | bytes | Path, destination: str | bytes | P result = commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert result.status == "rolled_back" @@ -520,7 +543,10 @@ def restore_then_concurrently_update(snapshot_to_restore, written_hashes): result = commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert result.status == "rollback_failed" @@ -551,7 +577,10 @@ def fail_write_and_first_restore( result = commit_prompt_bundle( snapshot, - {"system": "candidate system", "user": "candidate user"}, + { + "system": "candidate system", + "user": "candidate user" + }, ) assert result.status == "rollback_failed" diff --git a/trpc_agent_sdk/evaluation/_agent_optimizer.py b/trpc_agent_sdk/evaluation/_agent_optimizer.py index 7f907c3d..76641655 100644 --- a/trpc_agent_sdk/evaluation/_agent_optimizer.py +++ b/trpc_agent_sdk/evaluation/_agent_optimizer.py @@ -91,9 +91,7 @@ def _redact_config_secrets(value: Any) -> Any: if isinstance(value, dict): redacted = {} for key, child in value.items(): - normalized_key = "".join( - character for character in key.casefold() if character.isalnum() - ) + normalized_key = "".join(character for character in key.casefold() if character.isalnum()) if normalized_key.endswith(_SENSITIVE_CONFIG_KEY_SUFFIXES): redacted[key] = _REDACTED_CONFIG_VALUE else: From d7209c18c6de6633db3238ea9ad45dc37e47ff20 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 11 Jul 2026 11:40:41 +0800 Subject: [PATCH 34/34] fix(examples): redact camelcase audit secrets --- .../eval_optimize_loop/eval_loop/pipeline.py | 63 ++++++++++--------- .../tests/test_pipeline_orchestration.py | 41 ++++++++++-- 2 files changed, 69 insertions(+), 35 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py index bbded42c..f9878ff4 100644 --- a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py @@ -52,6 +52,36 @@ from .writeback import snapshot_prompt_files _REPOSITORY_ROOT = Path(__file__).resolve().parents[4] +_SENSITIVE_CONFIG_KEY_SUFFIXES = ( + "apikey", + "authorization", + "authtoken", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "idtoken", + "password", + "passwd", + "privatekey", + "refreshtoken", + "secret", + "secretaccesskey", + "secretkey", + "signingkey", + "sshkey", + "token", + "accesstoken", +) +_SENSITIVE_CONFIG_KEY_MARKERS = ( + "apikey", + "password", + "passwd", + "credential", + "secret", + "authorization", + "privatekey", +) @dataclass(frozen=True) @@ -1580,36 +1610,9 @@ def _pretty_json_sha256(value: Any) -> str: def _is_secret_key(lowered: str) -> bool: - normalized = lowered.replace("-", "_").replace(" ", "_") - if normalized in { - "api_key", - "apikey", - "token", - "access_token", - "refresh_token", - "auth_token", - "password", - "passwd", - "credentials", - "credential", - "secret", - "authorization", - "private_key", - "signing_key", - "ssh_key", - }: - return True - if normalized.endswith("_token"): - return True - return any(marker in normalized for marker in ( - "api_key", - "password", - "passwd", - "credential", - "secret", - "authorization", - "private_key", - )) + normalized = "".join(character for character in lowered if character.isalnum()) + return normalized.endswith(_SENSITIVE_CONFIG_KEY_SUFFIXES) or any(marker in normalized + for marker in _SENSITIVE_CONFIG_KEY_MARKERS) def _sanitize_public_value(value: Any) -> Any: diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py index 1bb7ba4f..14b56567 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py @@ -1681,11 +1681,19 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat "api_key": "plain-api-key", "nested": { "access_token": "plain-token", + "accessToken": "plain-camel-access-token", + "authToken": "plain-camel-auth-token", + "bearerToken": "plain-camel-bearer-token", + "idToken": "plain-camel-id-token", + "githubToken": "plain-camel-github-token", + "passwordValue": "plain-camel-password-value", "credentials": { "password": "plain-password" }, "github_token": "plain-github-token", "private_key": "plain-private-key", + "max_tokens": 2048, + "token_budget": 4096, "cache_path": str(tmp_path / "private-cache"), }, } @@ -1700,6 +1708,14 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat assert "plain-password" not in payload assert "plain-github-token" not in payload assert "plain-private-key" not in payload + assert "plain-camel-access-token" not in payload + assert "plain-camel-auth-token" not in payload + assert "plain-camel-bearer-token" not in payload + assert "plain-camel-id-token" not in payload + assert "plain-camel-github-token" not in payload + assert "plain-camel-password-value" not in payload + assert report.audit["config_snapshot"]["nested"]["max_tokens"] == 2048 + assert report.audit["config_snapshot"]["nested"]["token_budget"] == 4096 assert str(tmp_path.resolve()) not in payload assert report.run["reproducibility_shell"] == "powershell" assert "$EXTERNAL" in report.run["reproducibility_command"] @@ -1713,11 +1729,26 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat request.run_id, "config.snapshot.json", ).read_text(encoding="utf-8") - assert "plain-api-key" not in persisted + config_snapshot - assert "plain-token" not in persisted + config_snapshot - assert "plain-password" not in persisted + config_snapshot - assert "plain-github-token" not in persisted + config_snapshot - assert "plain-private-key" not in persisted + config_snapshot + final_run_dir = Path(request.output_dir) / "runs" / request.run_id + persisted_bytes = persisted.encode("utf-8") + persisted_markdown.encode("utf-8") + persisted_bytes += b"".join(path.read_bytes() for path in final_run_dir.rglob("*") if path.is_file()) + for secret in ( + "plain-api-key", + "plain-token", + "plain-password", + "plain-github-token", + "plain-private-key", + "plain-camel-access-token", + "plain-camel-auth-token", + "plain-camel-bearer-token", + "plain-camel-id-token", + "plain-camel-github-token", + "plain-camel-password-value", + ): + assert secret.encode("utf-8") not in persisted_bytes + redacted_snapshot = json.loads(config_snapshot) + assert redacted_snapshot["nested"]["max_tokens"] == 2048 + assert redacted_snapshot["nested"]["token_budget"] == 4096 assert "```powershell" in persisted_markdown assert str(tmp_path.resolve()) not in persisted_markdown assert hashlib.sha256(