diff --git a/.gitignore b/.gitignore index 233248dd..1e9b3f83 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ *.lock *.log examples/*.log +examples/optimization/eval_optimize_loop/outputs/optimization_report.json +examples/optimization/eval_optimize_loop/outputs/optimization_report.md +examples/optimization/eval_optimize_loop/outputs/runs/ trpc-agent-py.egg-info diff --git a/docs/superpowers/plans/2026-07-10-issue-91-safe-closed-loop.md b/docs/superpowers/plans/2026-07-10-issue-91-safe-closed-loop.md new file mode 100644 index 00000000..a7abd5fb --- /dev/null +++ b/docs/superpowers/plans/2026-07-10-issue-91-safe-closed-loop.md @@ -0,0 +1,1884 @@ +# Issue #91 Safe Closed-Loop Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build one auditable Evaluation + Optimization pipeline that satisfies Issue #91, rejects incomplete or regressing candidates, and writes source prompts only after the complete gate accepts them. + +**Architecture:** Move orchestration from `run_pipeline.py` into a shared async pipeline used by both fake and SDK backends. Backends return the same complete schemas; the pipeline owns evaluation order, gate decisions, run-specific audit persistence, and post-gate transactional writeback. + +**Tech Stack:** Python 3.11, dataclasses, asyncio, pathlib, `AgentEvaluator`, `AgentOptimizer`, `TargetPrompt`, pytest, YAPF, flake8. + +--- + +## Working-tree rules + +This worktree already contains intentional uncommitted changes in 17 Issue #91 files. Preserve them. Do not reset, restore, stash, or bulk-stage the worktree. Every task below stages only the listed files and commits one coherent change. + +### Task 1: Establish the baseline and lock down data contracts + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/schemas.py:12-157` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/loader.py:14-142` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/config.py:53-155` +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:771-814` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_config_validation.py` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py` + +- [ ] **Step 1: Record the current baseline without changing files** + +Run: + +```powershell +git status --short +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +``` + +Expected: the pre-existing Issue #91 files remain modified and the current suite reports `80 passed`. + +- [ ] **Step 2: Add failing strict-input and path-collision tests** + +Append these tests to `test_config_validation.py`: + +```python +import math + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.config import parse_optimizer_config +from examples.optimization.eval_optimize_loop.eval_loop.loader import read_json +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.run_pipeline import _load_sdk_gate_config + + +@pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity"]) +def test_read_json_rejects_non_standard_constants(tmp_path: Path, constant: str): + path = tmp_path / "bad.json" + path.write_text('{"value": ' + constant + "}", encoding="utf-8") + + with pytest.raises(ValueError, match="non-standard JSON constant"): + read_json(path) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf"), True]) +def test_fake_gate_rejects_non_finite_or_boolean_numbers(value): + with pytest.raises(ValueError, match="finite number"): + parse_optimizer_config( + {"gate": {"min_val_score_improvement": value}}, + path="optimizer.json", + ) + + +def test_sdk_gate_allows_explicitly_disabled_cost_limit(tmp_path: Path): + path = tmp_path / "gate.json" + path.write_text('{"gate": {"max_total_cost": null}}', encoding="utf-8") + + config = _load_sdk_gate_config(path) + + assert config["max_total_cost"] is None + + +def test_eval_case_rejects_explicit_split_mismatch(): + with pytest.raises(ValueError, match="split mismatch"): + EvalCase.from_dict( + { + "id": "case-1", + "split": "train", + "input": "Return OK", + "expectation": {"type": "exact", "expected": "OK"}, + }, + split="validation", + ) +``` + +Append this test to `test_sdk_backend.py`: + +```python +def test_target_prompt_paths_reject_same_resolved_file(tmp_path: Path): + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="same resolved file"): + _parse_target_prompt_paths( + [ + f"system_prompt={prompt_path}", + f"router_prompt={prompt_path.parent / '.' / prompt_path.name}", + ], + default_prompt_path=prompt_path, + ) +``` + +- [ ] **Step 3: Run the new tests and confirm the current implementation fails** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_config_validation.py examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py::test_target_prompt_paths_reject_same_resolved_file -v +``` + +Expected: failures show that Python JSON accepts non-standard constants, fake gate numbers accept `NaN`, split mismatch is not rejected, and aliased target paths are accepted. + +- [ ] **Step 4: Extend the shared schemas without breaking existing callers** + +In `schemas.py`, add defaults after the existing non-default `CaseResult` fields and add the new audit models: + +```python +from typing import Any +from typing import Literal + + +@dataclass(frozen=True) +class CaseResult: + case_id: str + split: str + score: float + passed: bool + output: str + metrics: dict[str, float] = field(default_factory=dict) + trace: dict[str, Any] = field(default_factory=dict) + trace_available: bool = False + failure_category: str | None = None + failure_reason: str | None = None + evidence: str | None = None + cost: float = 0.0 + hard_failed: bool = False + expected_failure_category: str | None = None + + +@dataclass(frozen=True) +class CandidatePrompt: + candidate_id: str + prompt: str + rationale: str + prompt_diff: str + prompt_fields: dict[str, str] = field(default_factory=dict) + + def bundle(self) -> dict[str, str]: + if self.prompt_fields: + return dict(self.prompt_fields) + return {"system_prompt": self.prompt} + + +@dataclass(frozen=True) +class CostSummary: + optimizer: float = 0.0 + evaluator: float = 0.0 + agent: float = 0.0 + total: float = 0.0 + complete: bool = True + + +@dataclass(frozen=True) +class OptimizationRound: + round_id: str + candidate_id: str + prompts: dict[str, str] + rationale: str + metrics: dict[str, float] + cost: float + duration_seconds: float + + +WritebackStatus = Literal[ + "rejected", + "not_requested", + "applied", + "rolled_back", + "rollback_failed", +] + + +@dataclass(frozen=True) +class WritebackResult: + status: WritebackStatus + before_hashes: dict[str, str] = field(default_factory=dict) + after_hashes: dict[str, str] = field(default_factory=dict) + error: str | None = None + + +@dataclass(frozen=True) +class OptimizationResult: + candidates: list[CandidatePrompt] + rounds: list[OptimizationRound] + cost: CostSummary + raw_summary: dict[str, Any] = field(default_factory=dict) +``` + +Add `rounds`, `cost_summary`, and `writeback` to the end of `OptimizationReport`, with defaults so existing report construction remains valid during the refactor: + +```python + rounds: list[OptimizationRound] = field(default_factory=list) + cost_summary: CostSummary = field(default_factory=CostSummary) + writeback: WritebackResult = field( + default_factory=lambda: WritebackResult(status="not_requested") + ) +``` + +- [ ] **Step 5: Implement strict JSON, finite-number, split, and path validation** + +Replace `read_json()` in `loader.py` with: + +```python +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-standard JSON constant is not allowed: {value}") + + +def read_json(path: str | Path) -> dict[str, Any]: + resolved = Path(path) + try: + with resolved.open("r", encoding="utf-8") as file: + payload = json.load(file, parse_constant=_reject_json_constant) + except (json.JSONDecodeError, ValueError) as exc: + raise ValueError(f"invalid strict JSON in {resolved}: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object in {resolved}") + return payload +``` + +In `config.py`, use one helper for every numeric gate field: + +```python +def _finite_number(value: Any, *, field_name: str, minimum: float, maximum: float | None = None) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field_name} must be a finite number") + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"{field_name} must be a finite number") + if parsed < minimum or (maximum is not None and parsed > maximum): + upper = f" and <= {maximum}" if maximum is not None else "" + raise ValueError(f"{field_name} must be >= {minimum}{upper}") + return parsed +``` + +Use it for `min_val_score_improvement` and `max_score_drop_per_case`. Keep fake `GateConfig.max_total_cost` defaulted to `1.0`, but type it as `float | None`; accept `None` as “cost gate disabled” and otherwise validate it with `_finite_number()`. Apply the same rule in `_load_sdk_gate_config()`. In `EvalCase.from_dict()`, reject an explicit payload split that differs from the loader split. In `_parse_target_prompt_paths()`, track `Path(path).resolve()` values and raise `ValueError("--target-prompt fields must not reference the same resolved file")` on duplicates. + +- [ ] **Step 6: Run focused and full example tests** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_config_validation.py examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py::test_target_prompt_paths_reject_same_resolved_file -v +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +``` + +Expected: all tests pass and no existing example test regresses. + +- [ ] **Step 7: Commit the contract and validation change** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/schemas.py examples/optimization/eval_optimize_loop/eval_loop/loader.py examples/optimization/eval_optimize_loop/eval_loop/config.py examples/optimization/eval_optimize_loop/run_pipeline.py examples/optimization/eval_optimize_loop/tests/test_config_validation.py examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +git commit -m "refactor(examples): define safe optimization contracts" +``` + +### Task 2: Add transactional prompt snapshots and post-gate writeback + +**Files:** +- Create: `examples/optimization/eval_optimize_loop/eval_loop/writeback.py` +- Create: `examples/optimization/eval_optimize_loop/tests/test_writeback.py` + +- [ ] **Step 1: Write failing snapshot, rollback, and compare-and-swap tests** + +Create `test_writeback.py`: + +```python +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ConcurrentPromptUpdateError +from examples.optimization.eval_optimize_loop.eval_loop.writeback import commit_prompt_bundle +from examples.optimization.eval_optimize_loop.eval_loop.writeback import snapshot_prompt_files +from examples.optimization.eval_optimize_loop.eval_loop.writeback import temporary_prompt_bundle + + +def _files(tmp_path: Path) -> dict[str, Path]: + system = tmp_path / "system.txt" + router = tmp_path / "router.txt" + system.write_text("system baseline", encoding="utf-8") + router.write_text("router baseline", encoding="utf-8") + return {"system_prompt": system, "router_prompt": router} + + +def test_temporary_prompt_bundle_always_restores_original_bytes(tmp_path: Path): + paths = _files(tmp_path) + snapshot = snapshot_prompt_files(paths) + + with pytest.raises(RuntimeError, match="candidate failed"): + with temporary_prompt_bundle( + snapshot, + {"system_prompt": "candidate system", "router_prompt": "candidate router"}, + ): + assert paths["system_prompt"].read_text(encoding="utf-8") == "candidate system" + raise RuntimeError("candidate failed") + + assert paths["system_prompt"].read_bytes() == b"system baseline" + assert paths["router_prompt"].read_bytes() == b"router baseline" + + +def test_commit_rejects_concurrent_source_change(tmp_path: Path): + paths = _files(tmp_path) + snapshot = snapshot_prompt_files(paths) + paths["system_prompt"].write_text("changed by another process", encoding="utf-8") + + with pytest.raises(ConcurrentPromptUpdateError): + commit_prompt_bundle( + snapshot, + {"system_prompt": "candidate system", "router_prompt": "candidate router"}, + ) + + assert paths["system_prompt"].read_text(encoding="utf-8") == "changed by another process" + + +def test_second_replace_failure_rolls_back_first_file(tmp_path: Path, monkeypatch): + paths = _files(tmp_path) + snapshot = snapshot_prompt_files(paths) + real_replace = os.replace + calls = 0 + + def fail_second_replace(source, destination): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("second replace failed") + return real_replace(source, destination) + + monkeypatch.setattr(os, "replace", fail_second_replace) + result = commit_prompt_bundle( + snapshot, + {"system_prompt": "candidate system", "router_prompt": "candidate router"}, + ) + + assert result.status == "rolled_back" + assert paths["system_prompt"].read_bytes() == b"system baseline" + assert paths["router_prompt"].read_bytes() == b"router baseline" +``` + +- [ ] **Step 2: Run the tests and confirm the module is missing** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_writeback.py -v +``` + +Expected: collection fails because `eval_loop.writeback` does not exist. + +- [ ] **Step 3: Implement the writeback module** + +Create `writeback.py` with these public types and functions: + +```python +from __future__ import annotations + +import hashlib +import os +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +from .schemas import WritebackResult + + +class ConcurrentPromptUpdateError(RuntimeError): + pass + + +@dataclass(frozen=True) +class PromptFileSnapshot: + name: str + path: Path + content: bytes + sha256: str + + +@dataclass(frozen=True) +class PromptSnapshot: + files: dict[str, PromptFileSnapshot] + + def hashes(self) -> dict[str, str]: + return {name: item.sha256 for name, item in self.files.items()} + + +def _hash_bytes(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def snapshot_prompt_files(paths: dict[str, str | Path]) -> PromptSnapshot: + files = {} + for name, raw_path in paths.items(): + path = Path(raw_path) + content = path.read_bytes() + files[name] = PromptFileSnapshot( + name=name, + path=path, + content=content, + sha256=_hash_bytes(content), + ) + return PromptSnapshot(files=files) + + +def _current_hashes(snapshot: PromptSnapshot) -> dict[str, str]: + return { + name: _hash_bytes(item.path.read_bytes()) + for name, item in snapshot.files.items() + } + + +def _atomic_replace_bytes(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temp_path = Path(temp_name) + try: + with os.fdopen(descriptor, "wb") as file: + file.write(content) + file.flush() + os.fsync(file.fileno()) + os.replace(temp_path, path) + finally: + if temp_path.exists(): + temp_path.unlink() + + +def _restore_snapshot(snapshot: PromptSnapshot) -> list[str]: + failures = [] + for name, item in snapshot.files.items(): + try: + _atomic_replace_bytes(item.path, item.content) + except OSError as exc: + failures.append(f"{name}: {exc}") + return failures + + +@contextmanager +def temporary_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> Iterator[None]: + missing = sorted(set(snapshot.files) - set(prompts)) + if missing: + raise ValueError(f"candidate prompt bundle is missing fields: {missing}") + try: + for name, item in snapshot.files.items(): + _atomic_replace_bytes(item.path, prompts[name].encode("utf-8")) + yield + finally: + failures = _restore_snapshot(snapshot) + if failures: + raise RuntimeError(f"failed to restore prompt snapshot: {failures}") + if _current_hashes(snapshot) != snapshot.hashes(): + raise RuntimeError("restored prompt hashes do not match the original snapshot") + + +def commit_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> WritebackResult: + before_hashes = _current_hashes(snapshot) + if before_hashes != snapshot.hashes(): + raise ConcurrentPromptUpdateError("source prompts changed after the run snapshot") + try: + for name, item in snapshot.files.items(): + _atomic_replace_bytes(item.path, prompts[name].encode("utf-8")) + except (OSError, KeyError) as exc: + rollback_failures = _restore_snapshot(snapshot) + status = "rollback_failed" if rollback_failures else "rolled_back" + return WritebackResult( + status=status, + before_hashes=before_hashes, + after_hashes=_current_hashes(snapshot), + error=f"{exc}; rollback_failures={rollback_failures}", + ) + return WritebackResult( + status="applied", + before_hashes=before_hashes, + after_hashes=_current_hashes(snapshot), + ) +``` + +- [ ] **Step 4: Run writeback tests** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_writeback.py -v +``` + +Expected: all three tests pass. + +- [ ] **Step 5: Commit the transactional writeback component** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/writeback.py examples/optimization/eval_optimize_loop/tests/test_writeback.py +git commit -m "feat(examples): add transactional prompt writeback" +``` + +### Task 3: Normalize fake and SDK backend contracts + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/backends.py` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/evaluator.py` + +- [ ] **Step 1: Replace the unsafe writeback expectation and add result-mapping tests** + +Replace `test_sdk_backend_passes_update_source_true` with: + +```python +def test_sdk_backend_never_delegates_source_writeback(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["update_source"] is False + assert prompt_path.read_text(encoding="utf-8") == "baseline" +``` + +Add a mapping test using small fake SDK result objects: + +```python +def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): + metric = SimpleNamespace( + metric_name="final_response_avg_score", + score=0.5, + eval_status="FAILED", + details=SimpleNamespace(reason="response mismatch"), + ) + invocation = SimpleNamespace( + actual_invocation=SimpleNamespace( + user_content={"parts": [{"text": "query"}]}, + final_response={"parts": [{"text": "actual"}]}, + intermediate_data={"tool_calls": [{"name": "lookup"}]}, + ) + ) + run = SimpleNamespace( + final_eval_status="FAILED", + overall_eval_metric_results=[metric], + eval_metric_result_per_invocation=[invocation], + error_message=None, + ) + result = SimpleNamespace( + results_by_eval_set_id={ + "set": SimpleNamespace(eval_results_by_eval_id={"case-1": [run]}) + } + ) + expected = EvalCase( + case_id="case-1", + split="validation", + input="query", + expectation={"type": "exact", "expected": "expected"}, + expected_failure_category="final_response_mismatch", + ) + + converted = _eval_result_from_sdk_result( + result, + prompt_id="candidate", + split="validation", + expected_cases={"case-1": expected}, + ) + + case = converted.cases[0] + assert case.metrics == {"final_response_avg_score": 0.5} + assert case.trace_available is True + assert case.trace["final_response"] == "actual" + assert case.expected_failure_category == "final_response_mismatch" +``` + +- [ ] **Step 2: Run the focused SDK tests and confirm failures** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py -k "never_delegates or preserves_metrics" -v +``` + +Expected: the old backend passes `update_source=True`, and the SDK converter lacks the new arguments and fields. + +- [ ] **Step 3: Make backend operations async and return `OptimizationResult`** + +In `backends.py`, define protocols and make both backends implement them: + +```python +from typing import Protocol + +from .schemas import OptimizationResult + + +class EvaluationBackend(Protocol): + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + raise NotImplementedError + + +class OptimizationBackend(Protocol): + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + raise NotImplementedError +``` + +Keep the current synchronous `SDKBackend.optimize()` only as a compatibility wrapper around `optimize_async()`. Remove the `update_source` field from `SDKBackend`; in `optimize_async()` hard-code: + +```python + result = await AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target_prompt, + train_dataset_path=str(train_path), + validation_dataset_path=str(val_path), + output_dir=str(output_dir), + update_source=False, + verbose=0, + ) +``` + +Extract all unique round candidates and the final best prompt into `CandidatePrompt.prompt_fields`; use IDs `sdk_round_001`, `sdk_round_002`, and `sdk_best`. Preserve `acceptance_reason`, metric breakdown, round cost, and duration in `OptimizationRound`. + +- [ ] **Step 4: Preserve per-metric results and invocation trace** + +Change `_eval_result_from_sdk_result()` to accept `expected_cases: dict[str, EvalCase]`. For each eval ID: + +```python +def _metric_scores(runs: list[Any]) -> dict[str, float]: + values: dict[str, list[float]] = {} + for run in runs: + for metric in getattr(run, "overall_eval_metric_results", []) or []: + score = getattr(metric, "score", None) + if isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(float(score)): + values.setdefault(str(metric.metric_name), []).append(float(score)) + return {name: round(sum(items) / len(items), 6) for name, items in values.items()} + + +def _invocation_trace(runs: list[Any]) -> tuple[dict[str, Any], bool]: + if not runs: + return {}, False + invocation_results = getattr(runs[-1], "eval_metric_result_per_invocation", []) or [] + if not invocation_results: + return {}, False + actual = getattr(invocation_results[-1], "actual_invocation", None) + if actual is None: + return {}, False + return { + "user_content": _safe_jsonable(getattr(actual, "user_content", None)), + "final_response": _content_text(getattr(actual, "final_response", None)), + "intermediate_data": _safe_jsonable(getattr(actual, "intermediate_data", None)), + }, True +``` + +Set `CaseResult.score` to the mean of the metric map, preserve `expected_failure_category`, and validate that the SDK result case IDs exactly match `expected_cases`. + +- [ ] **Step 5: Run backend tests and the example suite** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py -v +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +``` + +Expected: all tests pass; tests that previously asserted delegated source writes now assert `False`. + +- [ ] **Step 6: Commit the backend normalization** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/backends.py examples/optimization/eval_optimize_loop/eval_loop/evaluator.py examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py +git commit -m "refactor(examples): normalize optimization backends" +``` + +### Task 4: Remove fake-model oracle and split leakage + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/fake_model.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/optimizer.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/backends.py` +- Modify: `examples/optimization/eval_optimize_loop/data/train.evalset.json` +- Modify: `examples/optimization/eval_optimize_loop/data/val.evalset.json` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py` + +- [ ] **Step 1: Replace oracle-dependent tests with metamorphic tests** + +Replace `test_fake_model_generalization.py` with tests that hold user input and prompt constant while changing evaluator-only fields: + +```python +from __future__ import annotations + +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.optimizer import FakeOptimizer +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult + + +def _case(*, split: str, expected: str, protected: bool) -> EvalCase: + return EvalCase( + case_id=f"case-{split}-{expected}", + split=split, + input="Return only PUBLIC; do not use JSON.", + expectation={"type": "exact", "expected": expected}, + protected=protected, + tags=["evaluator-only"], + ) + + +def test_fake_model_output_does_not_change_with_oracle_fields(): + model = FakeModel(seed=91) + first, _, _ = model.generate("candidate", "baseline prompt", _case(split="train", expected="SECRET", protected=False)) + second, _, _ = model.generate("candidate", "baseline prompt", _case(split="validation", expected="OTHER", protected=True)) + + assert first == "PUBLIC" + assert second == "PUBLIC" + + +def test_fake_optimizer_requires_observed_training_failures(): + passing = EvalResult( + prompt_id="baseline", + split="train", + score=1.0, + passed=True, + cost=0.0, + cases=[CaseResult(case_id="c1", split="train", score=1.0, passed=True, output="OK")], + ) + failing = EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[ + CaseResult( + case_id="c1", + split="train", + score=0.0, + passed=False, + output="bad", + failure_category="format_violation", + failure_reason="not strict JSON", + ) + ], + ) + + optimizer = FakeOptimizer() + assert optimizer.propose("baseline", passing, {"by_category": {}}) == [] + candidates = optimizer.propose( + "baseline", + failing, + {"by_category": {"format_violation": 1}}, + ) + assert [candidate.candidate_id for candidate in candidates] == [ + "candidate_001_overfit", + "candidate_002_safe", + ] +``` + +- [ ] **Step 2: Run the metamorphic tests and confirm leakage** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py -v +``` + +Expected: outputs differ when split, protected, or expected value changes, and `FakeOptimizer.propose()` has the old signature. + +- [ ] **Step 3: Make fake behavior depend only on input and prompt** + +Replace expectation-driven output helpers in `fake_model.py` with input parsers: + +```python +import json +import re + + +ASSIGNMENT_RE = re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*)=([A-Za-z0-9_-]+)\b") +ONLY_RE = re.compile(r"return only\s+([A-Za-z0-9_-]+)", re.IGNORECASE) + + +def _assignments(text: str) -> dict[str, str]: + return {key: value for key, value in ASSIGNMENT_RE.findall(text)} + + +def _only_value(text: str) -> str | None: + match = ONLY_RE.search(text) + return match.group(1) if match else None + + +class FakeModel: + COST_PER_CALL = 0.001 + + def __init__(self, seed: int = 91) -> None: + self.seed = seed + + def generate(self, prompt_id: str, prompt: str, case: EvalCase) -> tuple[str, dict[str, Any], float]: + mode = self._mode(prompt) + output = self._render(mode, case.input) + return output, { + "seed": self.seed, + "prompt_id": prompt_id, + "prompt_mode": mode, + "case_id": case.case_id, + }, self.COST_PER_CALL + + def _mode(self, prompt: str) -> str: + if "Always force every final answer into JSON" in prompt: + return "overfit" + if "Use strict JSON only when the user explicitly asks" in prompt: + return "safe" + return "baseline" + + def _render(self, mode: str, user_input: str) -> str: + assignments = _assignments(user_input) + only_value = _only_value(user_input) + asks_json = "strict json" in user_input.lower() + natural = self._natural_answer(user_input) + if mode == "overfit": + payload = assignments or {"answer": only_value or natural} + return json.dumps(payload, sort_keys=True) + if asks_json: + payload = json.dumps(assignments, sort_keys=True) + return payload if mode == "safe" else f"Here is the JSON you requested: {payload}" + if only_value is not None: + return only_value + return natural + + def _natural_answer(self, user_input: str) -> str: + lowered = user_input.lower() + if "latency" in lowered and "retries" in lowered: + return "Latency and retries need monitoring." + if "cache" in lowered and "stale data" in lowered: + return "Cache invalidation prevents stale data." + return "The request was handled naturally." +``` + +Delete all reads of `case.expectation`, `case.split`, `case.protected`, `case.tags`, and `case.simulated_outputs` from `FakeModel`. + +- [ ] **Step 4: Make fake optimization consume baseline failures** + +Change `FakeOptimizer.propose()` to: + +```python +def propose( + self, + baseline_prompt: str, + baseline_train: EvalResult, + failure_summary: dict[str, object], +) -> list[CandidatePrompt]: + failed_categories = { + case.failure_category + for case in baseline_train.cases + if not case.passed and case.failure_category + } + summarized = set((failure_summary.get("by_category") or {}).keys()) + if not (failed_categories | summarized): + return [] + if not ((failed_categories | summarized) & {"format_violation", "final_response_mismatch"}): + return [] + return self._format_candidates(baseline_prompt) +``` + +Move the two existing candidate constructors into `_format_candidates()`, remove `OPTIMIZER_MARKER`, and keep the natural-language instructions used by `FakeModel._mode()`. + +- [ ] **Step 5: Update the six public user inputs without exposing judge labels to the model** + +Keep official SDK EvalSet shape. Use these user texts and evaluator expectations: + +```json +{ + "train": [ + ["Return strict JSON with intent=refund and priority=high.", {"type": "json", "expected_values": {"intent": "refund", "priority": "high"}}], + ["Return strict JSON with status=READY and next_step=ship.", {"type": "json", "expected_values": {"status": "READY", "next_step": "ship"}}], + ["Explain latency and retries naturally in under 80 characters.", {"type": "rubric", "must_include": ["latency", "retries"], "max_chars": 80}] + ], + "validation": [ + ["Return strict JSON with status=approved and next_step=email_customer.", {"type": "json", "expected_values": {"status": "approved", "next_step": "email_customer"}}], + ["Explain cache invalidation and stale data naturally.", {"type": "rubric", "must_include": ["cache", "stale data"], "forbidden": ["{", "}", "json"]}], + ["Return only YES; do not use JSON.", {"type": "exact", "expected": "YES"}] + ] +} +``` + +Mark the final validation case protected. This yields baseline train `1/3`, overfit train `2/3`, baseline validation `2/3`, overfit validation `1/3`, and safe validation `3/3` without reading split. + +- [ ] **Step 6: Run fake model and pipeline tests** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py -v +``` + +Expected: same input/prompt produces the same output across oracle-field changes; the overfit candidate is rejected and the safe candidate is selected. + +- [ ] **Step 7: Commit the oracle-free fake backend** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/fake_model.py examples/optimization/eval_optimize_loop/eval_loop/optimizer.py examples/optimization/eval_optimize_loop/eval_loop/backends.py examples/optimization/eval_optimize_loop/data/train.evalset.json examples/optimization/eval_optimize_loop/data/val.evalset.json examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py +git commit -m "fix(examples): remove fake evaluation oracle leakage" +``` + +### Task 5: Build the shared async pipeline and complete gate + +**Files:** +- Create: `examples/optimization/eval_optimize_loop/eval_loop/pipeline.py` +- Create: `examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/gate.py` +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/report.py` +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py` + +- [ ] **Step 1: Write orchestration tests with recording backends** + +Create `test_pipeline_orchestration.py` with the following complete recording backend. It uses temporary official-shaped input files and records every backend call without patching production code: + +```python +def _case(case_id: str, split: str, score: float) -> CaseResult: + return CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="OK" if score >= 1.0 else "FAIL", + metrics={"exact_match": score}, + trace={"final_response": "OK" if score >= 1.0 else "FAIL"}, + trace_available=True, + hard_failed=score == 0.0, + cost=0.01, + ) + + +def _result(prompt_id: str, split: str, scores: list[float]) -> EvalResult: + cases = [_case(f"{split[0]}{index}", split, score) for index, score in enumerate(scores)] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=sum(scores) / len(scores), + passed=all(case.passed for case in cases), + cost=sum(case.cost for case in cases), + cases=cases, + ) + + +class RecordingBackend: + def __init__( + self, + tmp_path: Path, + *, + candidate_regresses: bool = False, + cost_complete: bool = True, + ) -> None: + self.calls: list[tuple[str, ...]] = [] + self.candidate_regresses = candidate_regresses + self.cost_complete = cost_complete + self.train_path = DEFAULT_TRAIN + self.val_path = DEFAULT_VAL + self.config_path = DEFAULT_OPTIMIZER_CONFIG + self.prompt_path = tmp_path / "prompt.txt" + self.prompt_path.write_text("baseline prompt", encoding="utf-8") + + async def evaluate(self, *, prompt_id: str, split: str, **kwargs: Any) -> EvalResult: + self.calls.append(("evaluate", prompt_id, split)) + if prompt_id == "baseline": + scores = [0.0, 1.0, 1.0] + elif self.candidate_regresses and split == "validation": + scores = [0.0, 0.0, 1.0] + else: + scores = [1.0, 1.0, 1.0] + return _result(prompt_id, split, scores) + + async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: + self.calls.append(("optimize",)) + prompt = "candidate safe prompt" + candidate = CandidatePrompt( + candidate_id="candidate_safe", + prompt=prompt, + rationale="fix observed baseline failure", + prompt_diff="-baseline prompt\n+candidate safe prompt", + prompt_fields={"system_prompt": prompt}, + ) + return OptimizationResult( + candidates=[candidate], + rounds=[ + OptimizationRound( + round_id="round_001", + candidate_id=candidate.candidate_id, + prompts=candidate.bundle(), + rationale=candidate.rationale, + metrics={"discovery_score": 1.0}, + cost=0.02, + duration_seconds=0.01, + ) + ], + cost=CostSummary(optimizer=0.02, total=0.02, complete=self.cost_complete), + ) + + +async def _run_recording_pipeline(tmp_path: Path, backend: RecordingBackend) -> OptimizationReport: + return await run_pipeline_async( + train_path=backend.train_path, + val_path=backend.val_path, + optimizer_config_path=backend.config_path, + prompt_path=backend.prompt_path, + output_dir=tmp_path / "out", + mode="fake", + backend=backend, + run_id="recording-run", + ) + + +@pytest.mark.asyncio +async def test_pipeline_evaluates_baselines_before_optimization_and_every_candidate(tmp_path: Path): + backend = RecordingBackend(tmp_path) + report = await run_pipeline_async( + train_path=backend.train_path, + val_path=backend.val_path, + optimizer_config_path=backend.config_path, + prompt_path=backend.prompt_path, + output_dir=tmp_path / "out", + mode="fake", + backend=backend, + ) + + assert backend.calls == [ + ("evaluate", "baseline", "train"), + ("evaluate", "baseline", "validation"), + ("optimize",), + ("evaluate", "candidate_safe", "train"), + ("evaluate", "candidate_safe", "validation"), + ] + assert report.selected_candidate == "candidate_safe" + + +@pytest.mark.asyncio +async def test_gate_rejection_never_writes_source_even_when_requested(tmp_path: Path): + backend = RecordingBackend(tmp_path, candidate_regresses=True) + original = backend.prompt_path.read_bytes() + + report = await run_pipeline_async( + train_path=backend.train_path, + val_path=backend.val_path, + optimizer_config_path=backend.config_path, + prompt_path=backend.prompt_path, + output_dir=tmp_path / "out", + mode="fake", + update_source=True, + backend=backend, + ) + + assert report.selected_candidate is None + assert report.writeback.status == "rejected" + assert backend.prompt_path.read_bytes() == original + + +@pytest.mark.asyncio +async def test_incomplete_cost_rejects_when_budget_is_configured(tmp_path: Path): + backend = RecordingBackend(tmp_path, cost_complete=False) + report = await _run_recording_pipeline(tmp_path, backend) + + decision = report.gate_decisions[0] + assert decision.accepted is False + assert "cost_unavailable" in decision.reasons +``` + +Import `Any`, all schema classes used above, `DEFAULT_TRAIN`, `DEFAULT_VAL`, `DEFAULT_OPTIMIZER_CONFIG`, and `run_pipeline_async` at the top of the test. The backend method is named `optimize_candidates()` deliberately so it cannot be confused with the retained synchronous `SDKBackend.optimize()` compatibility wrapper. + +- [ ] **Step 2: Run orchestration tests and confirm the shared pipeline is absent** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py -v +``` + +Expected: collection fails because `run_pipeline_async` and `eval_loop.pipeline` do not exist. + +- [ ] **Step 3: Add the request model and shared candidate loop** + +Create `pipeline.py` with: + +```python +@dataclass(frozen=True) +class PipelineRequest: + train_path: Path + validation_path: Path + optimizer_config_path: Path + output_dir: Path + target_prompt_paths: dict[str, Path] + gate_config: dict[str, Any] + trace: bool + update_source: bool + mode: str + run_id: str + + +async def execute_pipeline( + request: PipelineRequest, + *, + evaluator: EvaluationBackend, + optimizer: OptimizationBackend, +) -> OptimizationReport: + started = time.perf_counter() + prompt_snapshot = snapshot_prompt_files(request.target_prompt_paths) + baseline_prompts = { + name: item.content.decode("utf-8") + for name, item in prompt_snapshot.files.items() + } + baseline_train = await evaluator.evaluate( + prompt_id="baseline", + prompts=baseline_prompts, + dataset_path=request.train_path, + split="train", + trace=request.trace, + artifact_dir=request.output_dir / "evaluator" / "baseline_train", + ) + baseline_validation = await evaluator.evaluate( + prompt_id="baseline", + prompts=baseline_prompts, + dataset_path=request.validation_path, + split="validation", + trace=request.trace, + artifact_dir=request.output_dir / "evaluator" / "baseline_validation", + ) + failure_summary = summarize_failures([baseline_train, baseline_validation]) + optimization = await optimizer.optimize_candidates( + baseline_prompts=baseline_prompts, + baseline_train=baseline_train, + failure_summary=failure_summary, + train_path=request.train_path, + validation_path=request.validation_path, + config_path=request.optimizer_config_path, + artifact_dir=request.output_dir / "optimizer", + ) + gate = AcceptanceGate(request.gate_config) + candidate_records = [] + deltas = [] + decisions = [] + cumulative_cost = round( + baseline_train.cost + baseline_validation.cost + optimization.cost.total, + 6, + ) + for candidate in optimization.candidates: + train_result = await evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=candidate.bundle(), + dataset_path=request.train_path, + split="train", + trace=request.trace, + artifact_dir=request.output_dir / "evaluator" / f"{candidate.candidate_id}_train", + ) + validation_result = await evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=candidate.bundle(), + dataset_path=request.validation_path, + split="validation", + trace=request.trace, + artifact_dir=request.output_dir / "evaluator" / f"{candidate.candidate_id}_validation", + ) + candidate_deltas = compute_case_deltas( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + ) + decision = gate.decide( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + deltas=candidate_deltas, + cumulative_cost=cumulative_cost, + cost_summary=optimization.cost, + ) + candidate_records.append({ + "candidate": candidate, + "train_result": train_result, + "validation_result": validation_result, + }) + deltas.extend(candidate_deltas) + decisions.append(decision) + cumulative_cost = decision.total_run_cost + selected_candidate = select_candidate(candidate_records, decisions) + duration_seconds = time.perf_counter() - started + explicit_evaluator_cost = sum( + result.cost + for result in [baseline_train, baseline_validation] + + [record[key] for record in candidate_records for key in ("train_result", "validation_result")] + ) + run_cost = replace( + optimization.cost, + evaluator=round(optimization.cost.evaluator + explicit_evaluator_cost, 6), + total=round(cumulative_cost, 6), + ) + report = build_report( + run={ + "run_id": request.run_id, + "mode": request.mode, + "trace": request.trace, + "update_source": request.update_source, + }, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + per_case_deltas=deltas, + gate_decisions=decisions, + selected_candidate=selected_candidate, + audit={ + "duration_seconds": duration_seconds, + "input_hashes": hash_pipeline_inputs(request, prompt_snapshot), + "cost": to_jsonable(run_cost), + }, + ) + return replace(report, rounds=optimization.rounds, cost_summary=run_cost) +``` + +Implement `hash_pipeline_inputs()` by hashing the train, validation, optimizer config, and snapshotted prompt bytes with SHA-256. Implement `select_candidate()` with the existing stable ordering: accepted candidates only, then highest validation score, then train score, then earliest candidate. + +- [ ] **Step 4: Make the gate reject incomplete data and incomplete configured cost** + +Add `cost_summary: CostSummary` to `AcceptanceGate.decide()`. Before score checks: + +```python +for baseline, candidate in ( + (baseline_train, candidate_train), + (baseline_validation, candidate_validation), +): + baseline_ids = [case.case_id for case in baseline.cases] + candidate_ids = [case.case_id for case in candidate.cases] + if len(baseline_ids) != len(set(baseline_ids)) or len(candidate_ids) != len(set(candidate_ids)): + reasons.append("reject: duplicate case IDs prevent a complete gate") + elif set(baseline_ids) != set(candidate_ids): + reasons.append("reject: baseline and candidate case IDs do not match") + +if self.config.get("max_total_cost") is not None and not cost_summary.complete: + reasons.append("reject: cost_unavailable for configured max_total_cost") +``` + +Treat `cumulative_cost` as the already-incurred baseline plus optimizer plus prior-candidate cost, and add only the current candidate's two explicit evaluation costs when calculating `total_run_cost`. `cost_summary` is passed separately to carry completeness; do not add its total a second time. Use the already-correct soft-to-hard condition: a candidate hard failure is new whenever the baseline case was not hard-failed. + +- [ ] **Step 5: Make `run_pipeline.py` a compatibility wrapper** + +Expose the full async signature and keep the sync API: + +```python +async def run_pipeline_async( + *, + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, + run_id: str | None = None, + backend: Any | None = None, +) -> OptimizationReport: + request, selected_backend = build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=output_dir, + mode=mode, + fake_model=fake_model, + fake_judge=fake_judge, + trace=trace, + sdk_call_agent=sdk_call_agent, + update_source=update_source, + gate_config_path=gate_config_path, + target_prompts=target_prompts, + run_id=run_id, + backend=backend, + ) + return await execute_pipeline( + request, + evaluator=selected_backend, + optimizer=selected_backend, + ) + + +def run_pipeline(**kwargs: Any) -> OptimizationReport: + if _has_running_loop(): + raise ValueError("run_pipeline() cannot run inside an active event loop; await run_pipeline_async()") + return asyncio.run(run_pipeline_async(**kwargs)) +``` + +Delete `_build_sdk_report()` and `_sdk_gate_decision()` after all callers move to `execute_pipeline()`. + +Define `build_pipeline_request_and_backend()` immediately below the wrapper. Its implementation must perform these exact operations in order: + +```python +def build_pipeline_request_and_backend(*, backend: Any | None, **options: Any) -> tuple[PipelineRequest, Any]: + mode = str(options["mode"]) + if mode not in {"fake", "sdk"}: + raise ValueError("mode must be 'fake' or 'sdk'") + prompt_path = Path(options["prompt_path"]) + target_paths = _parse_target_prompt_paths( + options.get("target_prompts"), + default_prompt_path=prompt_path, + ) + selected_run_id = validate_run_id(options.get("run_id") or create_run_id()) + if mode == "fake": + config = load_optimizer_config(options["optimizer_config_path"]) + gate_config = asdict(config.gate) + selected_backend = backend or FakeBackend( + fake_model=options["fake_model"], + fake_judge=options["fake_judge"], + ) + else: + if not options.get("sdk_call_agent") and backend is None: + raise ValueError("--sdk-call-agent is required in sdk mode") + gate_config = _load_sdk_gate_config(options.get("gate_config_path")) + selected_backend = backend or SDKBackend( + target_prompt_paths=target_paths, + call_agent_path=options["sdk_call_agent"], + ) + request = PipelineRequest( + train_path=Path(options["train_path"]), + validation_path=Path(options["val_path"]), + optimizer_config_path=Path(options["optimizer_config_path"]), + output_dir=Path(options["output_dir"]), + target_prompt_paths=target_paths, + gate_config=gate_config, + trace=bool(options["trace"]), + update_source=bool(options["update_source"]), + mode=mode, + run_id=selected_run_id, + ) + return request, selected_backend +``` + +Keep `create_run_id()` and `validate_run_id()` small and deterministic at the boundary: generated IDs use UTC timestamp plus a random suffix; supplied IDs must satisfy the report artifact-name regex. Adapt constructor keywords to the final backend classes, but do not add mode-specific orchestration outside this factory. + +- [ ] **Step 6: Persist audit before optional source write, then attach `WritebackResult`** + +At the end of `execute_pipeline()`, call `prepare_run_artifacts(report, request.output_dir)` before any final write. It creates the run-specific temporary directory, writes the complete pre-write report and audit payload with `allow_nan=False`, and returns `RunArtifactPaths`. Then: + +```python +artifact_paths = prepare_run_artifacts(report, request.output_dir) +if selected_candidate is None: + writeback = WritebackResult(status="rejected", before_hashes=prompt_snapshot.hashes()) +elif not request.update_source: + writeback = WritebackResult(status="not_requested", before_hashes=prompt_snapshot.hashes()) +else: + selected = next(item for item in optimization.candidates if item.candidate_id == selected_candidate) + writeback = commit_prompt_bundle(prompt_snapshot, selected.bundle()) +report = replace(report, writeback=writeback) +finalize_run_artifacts(report, artifact_paths) +return report +``` + +Use `artifact_paths = prepare_run_artifacts(...)` for the value passed to finalization. Task 6 supplies the exact `RunArtifactPaths`, preparation, and finalization implementations. This order is the regression guarantee: audit preparation must succeed before `commit_prompt_bundle()` is reachable. + +- [ ] **Step 7: Run orchestration, gate, fake, and SDK tests** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py examples/optimization/eval_optimize_loop/tests/test_gate.py -v +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +``` + +Expected: complete call order is enforced, rejected candidates never change source bytes, and no test expects `partial_applied`. + +- [ ] **Step 8: Commit the shared pipeline** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/pipeline.py examples/optimization/eval_optimize_loop/eval_loop/gate.py examples/optimization/eval_optimize_loop/eval_loop/report.py examples/optimization/eval_optimize_loop/run_pipeline.py examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py examples/optimization/eval_optimize_loop/tests/test_gate.py +git commit -m "refactor(examples): unify evaluation optimization pipeline" +``` + +### Task 6: Make report artifacts immutable and reproducible + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/eval_loop/report.py` +- Create: `examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py` +- Modify: `examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py` + +- [ ] **Step 1: Write failing run-isolation and sample-hash tests** + +Create `test_report_artifacts.py`: + +```python +def test_existing_run_id_is_never_overwritten(tmp_path: Path): + first = run_pipeline(output_dir=tmp_path, mode="fake", trace=True, run_id="fixed-run") + run_report = tmp_path / "runs" / "fixed-run" / "optimization_report.json" + original = run_report.read_bytes() + + with pytest.raises(FileExistsError, match="fixed-run"): + run_pipeline(output_dir=tmp_path, mode="fake", trace=True, run_id="fixed-run") + + assert run_report.read_bytes() == original + assert first.audit["duration_seconds"] > 0 + + +def test_committed_example_hashes_match_committed_inputs(): + root = Path("examples/optimization/eval_optimize_loop") + payload = json.loads((root / "outputs/optimization_report.example.json").read_text(encoding="utf-8")) + inputs = { + "train": root / "data/train.evalset.json", + "validation": root / "data/val.evalset.json", + "optimizer": root / "data/optimizer.json", + "prompt": root / "prompts/baseline_system_prompt.txt", + } + + for name, path in inputs.items(): + assert payload["audit"]["input_hashes"][name] == hashlib.sha256(path.read_bytes()).hexdigest() + serialized = json.dumps(payload, ensure_ascii=False) + assert "C:\\\\Users\\\\" not in serialized + assert "/Users/" not in serialized + assert "/home/" not in serialized +``` + +- [ ] **Step 2: Run the tests and observe overwrite/stale-hash failures** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py -v +``` + +Expected: fake mode deletes/reuses its run directory, duration is zero, and committed example hashes mismatch. + +- [ ] **Step 3: Implement run-specific temporary and final paths** + +In `report.py`, add: + +```python +@dataclass(frozen=True) +class RunArtifactPaths: + output_dir: Path + temporary: Path + final: Path + + +def create_run_artifact_paths(output_dir: str | Path, run_id: str) -> RunArtifactPaths: + root = Path(output_dir) + final = root / "runs" / _safe_artifact_name(run_id) + temporary = root / "runs" / f".{_safe_artifact_name(run_id)}.tmp" + if final.exists() or temporary.exists(): + raise FileExistsError(f"run id already exists: {run_id}") + temporary.mkdir(parents=True) + return RunArtifactPaths(output_dir=root, temporary=temporary, final=final) + + +def finalize_run_directory(paths: RunArtifactPaths) -> None: + paths.temporary.replace(paths.final) + + +def prepare_run_artifacts(report: OptimizationReport, output_dir: str | Path) -> RunArtifactPaths: + paths = create_run_artifact_paths(output_dir, str(report.run["run_id"])) + (paths.temporary / "optimization_report.json").write_text( + report_to_json(report), + encoding="utf-8", + ) + (paths.temporary / "optimization_report.md").write_text( + render_markdown(report), + encoding="utf-8", + ) + write_audit_artifacts(report, paths.temporary) + return paths + + +def _atomic_text(path: Path, value: str) -> None: + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(value, encoding="utf-8") + os.replace(temporary, path) + + +def finalize_run_artifacts(report: OptimizationReport, paths: RunArtifactPaths) -> None: + json_value = report_to_json(report) + markdown_value = render_markdown(report) + _atomic_text(paths.temporary / "optimization_report.json", json_value) + _atomic_text(paths.temporary / "optimization_report.md", markdown_value) + _atomic_text( + paths.temporary / "writeback.json", + json.dumps(to_jsonable(report.writeback), indent=2, sort_keys=True, allow_nan=False) + "\n", + ) + finalize_run_directory(paths) + _atomic_text(paths.output_dir / "optimization_report.json", json_value) + _atomic_text(paths.output_dir / "optimization_report.md", markdown_value) +``` + +Import `os`. Refactor `write_audit_artifacts()` so its second argument is the already-created run directory and it never creates, deletes, or reuses a run ID. Remove the fake-mode `shutil.rmtree(run_dir)` branch. Put SDK optimizer artifacts under the temporary run directory before finalization, not under a shared `/sdk_optimizer` directory. If preparation or finalization raises, do not reach source writeback; leave the temporary directory as failure evidence. + +- [ ] **Step 4: Serialize complete audit artifacts** + +Write baseline and every candidate split under `case_results/`; write `rounds/.json`, `per_case_deltas.json`, `gate_decisions.json`, `writeback.json`, prompt bundles, diffs, input hashes, and config snapshot. Use `allow_nan=False` for every JSON write. + +Normalize displayed paths with: + +```python +def display_path(path: str | Path, *, repository_root: Path) -> str: + resolved = Path(path).resolve() + try: + return resolved.relative_to(repository_root.resolve()).as_posix() + except ValueError: + return Path(path).as_posix() +``` + +Do not serialize the resolved contributor workspace path into committed examples. + +- [ ] **Step 5: Record real duration and keep latest convenience copies** + +Set `audit["duration_seconds"]` from the pipeline `perf_counter` measurement. After the run directory is finalized, atomically replace top-level `optimization_report.json` and `.md` with the finalized report contents. The immutable run directory remains authoritative. + +- [ ] **Step 6: Run reporting tests** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py -v +``` + +Expected: a duplicate run ID raises without modifying the first report, runtime is positive, and all audit JSON is strict. + +- [ ] **Step 7: Commit immutable audit artifacts** + +```powershell +git add examples/optimization/eval_optimize_loop/eval_loop/report.py examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py +git commit -m "fix(examples): make optimization audit reproducible" +``` + +### Task 7: Add independent acceptance evidence and real SDK integration + +**Files:** +- Create: `examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json` +- Create: `examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json` +- Create: `examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py` +- Create: `examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py` + +- [ ] **Step 1: Create an independent holdout decision fixture** + +Create `holdout_gate_cases.json` with ten labeled scenarios: + +```json +[ + {"id": "safe_gain", "expected": true, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 0.02, "budget": 1.0}, + {"id": "no_gain", "expected": false, "train": [1, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 0.02, "budget": 1.0}, + {"id": "overfit", "expected": false, "train": [0, 0, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 0, 0], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "validation_regression", "expected": false, "train": [1, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 1, 1], "candidate_validation": [1, 1, 0], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "protected_regression", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 0, 0], "candidate_validation": [0, 1, 1], "protected": ["v0"], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "new_hard_fail", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 0, 0], "candidate_validation": [0, 1, 1], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "soft_to_hard", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.5, 0, 0], "candidate_validation": [0, 1, 1], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "single_case_drop", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.7, 0, 0], "candidate_validation": [0.5, 1, 1], "protected": [], "max_drop": 0.1, "cost": 0.02, "budget": 1.0}, + {"id": "over_budget", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 1.1, "budget": 1.0}, + {"id": "safe_gain_with_soft_failure", "expected": true, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.5, 0, 1], "candidate_validation": [0.5, 1, 1], "protected": ["v0"], "max_drop": 0.0, "cost": 0.02, "budget": 1.0} +] +``` + +Create `attribution_cases.json` with labels separated from evidence: + +```json +[ + {"error_code": "json_parse_failure", "evidence": "JSON parser failed", "expected": "format_violation"}, + {"error_code": "required_key_missing", "evidence": "missing key status", "expected": "final_response_mismatch"}, + {"error_code": "exact_answer_mismatch", "evidence": "expected YES", "expected": "final_response_mismatch"}, + {"error_code": "tool_call_error", "evidence": "expected lookup", "expected": "tool_call_error"}, + {"error_code": "parameter_error", "evidence": "id mismatch", "expected": "parameter_error"}, + {"error_code": "missing_rubric_terms", "evidence": "missing latency", "expected": "llm_rubric_not_met"}, + {"error_code": "knowledge_recall_insufficient", "evidence": "missing doc-a", "expected": "knowledge_recall_insufficient"}, + {"error_code": "forbidden_pattern", "evidence": "forbidden JSON", "expected": "format_violation"} +] +``` + +- [ ] **Step 2: Implement threshold tests** + +In `test_acceptance_thresholds.py`, load fixtures, build `EvalResult` objects without exposing `expected` to the gate or attribution function, and assert: + +```python +def test_holdout_gate_decision_accuracy_is_at_least_eighty_percent(): + scenarios = _load_fixture("holdout_gate_cases.json") + correct = 0 + for scenario in scenarios: + decision = _decision_for_scenario(scenario) + correct += decision.accepted == scenario["expected"] + assert correct / len(scenarios) >= 0.80 + + +def test_independent_attribution_accuracy_is_at_least_seventy_five_percent(): + scenarios = _load_fixture("attribution_cases.json") + predictions = [attribute_failure(item["error_code"], item["evidence"]) for item in scenarios] + correct = sum(prediction[0] == item["expected"] for prediction, item in zip(predictions, scenarios)) + assert correct / len(scenarios) >= 0.75 + assert all(prediction[1] and prediction[2] for prediction in predictions) + + +def test_fake_trace_pipeline_finishes_under_three_minutes(tmp_path: Path): + started = time.perf_counter() + report = run_pipeline(output_dir=tmp_path, mode="fake", trace=True, run_id="performance") + elapsed = time.perf_counter() - started + assert elapsed < 180 + assert 0 < report.audit["duration_seconds"] <= elapsed +``` + +Use these fixture helpers in the same file; only the final accuracy comparison reads the `expected` label: + +```python +FIXTURES = Path(__file__).parent / "fixtures" + + +def _load_fixture(name: str) -> list[dict[str, Any]]: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def _eval(prompt_id: str, split: str, scores: list[float], *, cost: float = 0.0) -> EvalResult: + cases = [ + CaseResult( + case_id=f"{split[0]}{index}", + split=split, + score=float(score), + passed=score >= 1.0, + output=str(score), + metrics={"holdout": float(score)}, + hard_failed=score == 0.0, + ) + for index, score in enumerate(scores) + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=sum(scores) / len(scores), + passed=all(case.passed for case in cases), + cost=cost, + cases=cases, + ) + + +def _decision_for_scenario(scenario: dict[str, Any]) -> GateDecision: + baseline_train = _eval("baseline", "train", scenario["train"]) + baseline_validation = _eval("baseline", "validation", scenario["validation"]) + candidate_train = _eval("candidate", "train", scenario["candidate_train"]) + candidate_validation = _eval("candidate", "validation", scenario["candidate_validation"]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + ) + gate = AcceptanceGate({ + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": scenario["protected"], + "max_score_drop_per_case": scenario["max_drop"], + "max_total_cost": scenario["budget"], + }) + return gate.decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + deltas=deltas, + cumulative_cost=float(scenario["cost"]), + cost_summary=CostSummary(total=float(scenario["cost"]), complete=True), + ) +``` + +Import `Any`, `json`, all referenced schema/gate/report symbols, and `Path`. Here `scenario["cost"]` is supplied as already-incurred cost and the synthetic result costs are zero, matching the production rule that optimizer cost is counted once. + +- [ ] **Step 3: Add a real SDK facade/evaluator integration test** + +In `test_sdk_integration.py`, import the real `AgentOptimizer`, `AgentEvaluator`, `TargetPrompt`, and `GepaReflectiveOptimizer`. Monkeypatch only `GepaReflectiveOptimizer._call_gepa_optimize` with a deterministic GEPA result: + +```python +class FakeGEPAResult: + def __init__(self, baseline: dict[str, str], candidate: dict[str, str]): + self.candidates = [baseline, candidate] + self.val_aggregate_scores = [2 / 3, 1.0] + self.parents = [[None], [0]] + self.discovery_eval_counts = [0, 1] + self.total_metric_calls = 6 + self.best_outputs_valset = None + + @property + def best_idx(self) -> int: + return 1 + + +@pytest.mark.asyncio +async def test_sdk_pipeline_uses_real_facade_evaluator_and_post_gate_writeback(tmp_path: Path, monkeypatch): + prompt_path = tmp_path / "system_prompt.txt" + prompt_path.write_text("baseline prompt", encoding="utf-8") + gate_path = tmp_path / "gate.json" + gate_path.write_text('{"gate": {"max_total_cost": null}}', encoding="utf-8") + baseline = {"system_prompt": "baseline prompt"} + candidate = { + "system_prompt": "baseline prompt\nUse strict JSON only when the user explicitly asks." + } + + async def fake_call_gepa(self, **kwargs): + return FakeGEPAResult(baseline, candidate) + + async def call_agent(query: str) -> str: + prompt = prompt_path.read_text(encoding="utf-8") + return deterministic_sdk_response(prompt, query) + + monkeypatch.setattr(GepaReflectiveOptimizer, "_call_gepa_optimize", fake_call_gepa) + module = ModuleType("issue91_sdk_call_agent") + module.call_agent = call_agent + monkeypatch.setitem(sys.modules, module.__name__, module) + + report = await run_pipeline_async( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + sdk_call_agent="issue91_sdk_call_agent:call_agent", + gate_config_path=gate_path, + update_source=True, + run_id="sdk-integration", + ) + + assert report.selected_candidate is not None + assert all(record["train_result"].cases for record in report.candidates) + assert all(record["validation_result"].cases for record in report.candidates) + assert all(decision.gate_status == "applied" for decision in report.gate_decisions) + assert report.writeback.status == "applied" + assert prompt_path.read_text(encoding="utf-8") == candidate["system_prompt"] +``` + +Define the SDK response helper by routing only the prompt and user query through the oracle-free fake model. The dummy expectation proves the response path does not receive evaluator labels: + +```python +def deterministic_sdk_response(prompt: str, query: str) -> str: + case = EvalCase( + case_id="sdk-runtime-query", + split="runtime", + input=query, + expectation={"type": "runtime-only; must not be inspected"}, + ) + output, _, _ = FakeModel(seed=91).generate("sdk-runtime", prompt, case) + return output +``` + +Import `DEFAULT_TRAIN`, `DEFAULT_VAL`, and `DEFAULT_OPTIMIZER_CONFIG` from `run_pipeline`. The null cost limit is intentional because this deterministic facade test cannot account for external LM billing; the separate gate tests cover configured complete and incomplete costs. Do not patch `AgentEvaluator`, `AgentOptimizer`, or `TargetPrompt`; patch only the external GEPA call shown above. + +- [ ] **Step 4: Run acceptance and SDK integration tests** + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py -v +``` + +Expected: gate accuracy is at least 0.80, attribution accuracy is at least 0.75, performance is under 180 seconds, and the real SDK integration produces complete cases and post-gate writeback. + +- [ ] **Step 5: Commit acceptance evidence** + +```powershell +git add examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py +git commit -m "test(examples): prove issue 91 acceptance thresholds" +``` + +### Task 8: Regenerate examples, update documentation, and run merge verification + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/README.md` +- Modify: `examples/optimization/eval_optimize_loop/DESIGN.md` +- Modify: `examples/optimization/eval_optimize_loop/data/optimizer.json` +- Modify: `examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json` +- Modify: `examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md` +- Modify: `.gitignore` +- Modify: `pyproject.toml` + +- [ ] **Step 1: Update user documentation to match the unified semantics** + +Document these exact guarantees in README and the 300–500 字 design summary: + +- fake and SDK share baseline, candidate re-evaluation, delta, and gate semantics; +- SDK optimization always uses `update_source=False` internally; +- `--update-source` writes only the selected candidate after audit preparation and full gate acceptance; +- configured cost gate rejects when total cost is incomplete; +- fake model reads only user input and prompt; +- run directories are immutable and run-specific; +- SDK mode reports real per-case metrics and trace availability. + +Remove every statement describing `partial_applied`, aggregate-only accepted candidates, fixed zero duration, or shared SDK artifact directories. + +- [ ] **Step 2: Regenerate committed example outputs from the final fake pipeline** + +Run: + +```powershell +$output = Join-Path $env:TEMP 'issue91-final-example' +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --trace --run-id example --output-dir $output +Copy-Item -LiteralPath (Join-Path $output 'optimization_report.json') -Destination 'examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json' -Force +Copy-Item -LiteralPath (Join-Path $output 'optimization_report.md') -Destination 'examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md' -Force +``` + +Expected: selected candidate is the safe candidate, the overfit candidate is rejected, duration is positive, input paths are repository-relative, and hashes match current committed inputs. + +- [ ] **Step 3: Run example formatting and focused verification** + +```powershell +python -m compileall -q examples/optimization/eval_optimize_loop +python -m yapf --diff -r examples/optimization/eval_optimize_loop +python -m flake8 examples/optimization/eval_optimize_loop +python -m pytest examples/optimization/eval_optimize_loop/tests --tb=short +git diff --check +``` + +Expected: every command exits 0, YAPF emits no diff, and the example test suite has zero failures. + +- [ ] **Step 4: Run repository CI-equivalent tests and build** + +```powershell +python -m pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term --cov-fail-under=80 tests/ +python -m build +python -c "import trpc_agent_sdk; print('Import OK')" +``` + +Expected: repository tests pass with coverage at least 80%, package build succeeds, and the import command prints `Import OK`. + +- [ ] **Step 5: Verify the explicit Issue #91 merge checklist** + +Run: + +```powershell +python -m pytest examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py -v +rg -n "partial_applied|duration_seconds.: 0\.0|C:\\\\Users\\\\|/Users/|/home/" examples/optimization/eval_optimize_loop +git status --short +git diff --stat origin/main...HEAD +``` + +Expected: threshold tests pass; `rg` finds no stale partial-gate, zero-duration, or personal-path text in runtime/example artifacts; status lists only intentional final changes; diff scope remains limited to Issue #91 and the two superpowers documents. + +- [ ] **Step 6: Commit documentation and generated artifacts** + +```powershell +git add .gitignore pyproject.toml examples/optimization/eval_optimize_loop/README.md examples/optimization/eval_optimize_loop/DESIGN.md examples/optimization/eval_optimize_loop/data/optimizer.json examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md +git commit -m "docs(examples): finalize issue 91 closed loop" +``` + +- [ ] **Step 7: Perform the final branch review before publishing** + +```powershell +git log --oneline origin/main..HEAD +git diff --check origin/main...HEAD +git status --short --branch +``` + +Expected: commits are task-scoped, diff check exits 0, the working tree is clean, and the branch contains no unrelated files. Use `superpowers:requesting-code-review` before push or PR update. diff --git a/docs/superpowers/specs/2026-07-10-issue-91-safe-closed-loop-design.md b/docs/superpowers/specs/2026-07-10-issue-91-safe-closed-loop-design.md new file mode 100644 index 00000000..1d2a89d0 --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-issue-91-safe-closed-loop-design.md @@ -0,0 +1,245 @@ +# Issue #91 安全评测优化闭环设计 + +## 背景 + +PR #119 已经提供 fake 评测、候选 prompt、gate 和 JSON/Markdown 报告，但已发布提交中没有一条路径同时完成真实 baseline 评测、候选逐 case 复评、完整 gate 和 gate 后回写。fake model 还会读取 expectation、split、protected 等评测真值，因此现有测试不能证明隐藏样本决策准确率。SDK 模式把 `--update-source` 直接传给 `AgentOptimizer`，候选可能在 wrapper gate 拒绝前写入源 prompt。 + +本设计的边界是通过 Issue #91 的验收并安全合并。与验收无关的通用框架重构不纳入本次修改。 + +## 目标 + +1. fake 和 SDK 共用同一条“评测—归因—优化—复评—gate—审计—可选回写”流程。 +2. SDK 使用真实 `AgentEvaluator`、`AgentOptimizer` 和 `TargetPrompt`，但外部模型调用仍可在 CI 中替换为确定性实现。 +3. gate 只消费完整、可核验的逐 case 结果；不再允许 `cases=[]` 或 `partial_applied` 的候选被选中。 +4. 源 prompt 只在候选通过完整 gate 且用户显式指定 `--update-source` 后写回。 +5. 提供可执行证据证明隐藏决策准确率、失败归因准确率、过拟合拒绝和三分钟性能要求。 +6. 报告、输入哈希、耗时、成本和运行产物可以复核，不泄露贡献者机器的绝对路径。 + +## 非目标 + +- 不修改 `AgentEvaluator`、`AgentOptimizer` 或 GEPA 的公共 API。 +- 不引入新的远程服务、数据库或长期任务系统。 +- 不要求 CI 使用真实 API key。 +- 不把示例扩展成通用生产部署平台。 + +## 选定方案 + +采用统一闭环编排器。fake 与 SDK 仅负责各自的 evaluate/optimize 适配，候选选择、gate、审计和写回语义由共享 pipeline 实现。 + +分别修补两条路径虽然改动较少，但会保留两套不同的 gate 和报告语义；完全 SDK-first 会放大重构范围。统一编排器在验收覆盖、维护成本和合并风险之间更平衡，并可直接承接当前工作树中已有的 SDK EvalSet 与 AgentEvaluator 适配工作。 + +## 组件与文件职责 + +### `run_pipeline.py` + +保留 CLI、路径解析和同步兼容入口。新增异步核心入口的薄包装：同步 `run_pipeline()` 在没有活动事件循环时调用 `asyncio.run(run_pipeline_async(...))`；异步调用者直接使用 `run_pipeline_async()`。该文件不再实现候选循环、SDK 报告拼装或写回逻辑。 + +### `eval_loop/pipeline.py` + +新增共享编排器，负责： + +1. 冻结输入与 prompt 快照； +2. baseline train/validation 评测； +3. 失败归因； +4. 调用 optimizer backend； +5. 对每个候选重新执行 train/validation 评测； +6. 计算逐 case delta 和完整 gate； +7. 选择候选； +8. 写审计产物； +9. 在允许时执行安全回写。 + +### `eval_loop/backends.py` + +backend 统一暴露异步接口： + +```python +class EvaluationBackend(Protocol): + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: Path, + split: str, + trace: bool, + artifact_dir: Path, + ) -> EvalResult: ... + + +class OptimizationBackend(Protocol): + async def optimize( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: Path, + validation_path: Path, + config_path: Path, + artifact_dir: Path, + ) -> OptimizationResult: ... +``` + +`SDKBackend.optimize()` 始终调用 `AgentOptimizer.optimize(update_source=False)`。它从 `OptimizeResult.rounds[].candidate_prompts` 和 `best_prompts` 提取、去重候选；每个候选由共享 pipeline 做完整 train/validation 复评。`SDKBackend.evaluate()` 使用真实 `AgentEvaluator`，并把每个 case 的逐 metric 分数、pass/fail、原因、证据和可用 trace 转为统一 schema。关键轨迹从 `EvalCaseResult.eval_metric_result_per_invocation[].actual_invocation` 提取用户输入、工具调用和最终回复；SDK 没有提供某类轨迹时显式设置 `trace_available=False`，不能伪造空轨迹为“已采集”。 + +`FakeBackend.optimize()` 只接收 baseline train 结果、失败归因和 optimizer 配置，不接收 validation 评测结果。它根据观察到的失败类别生成确定性候选，而不是无条件返回固定候选。 + +### `eval_loop/gate.py` + +保留唯一 gate 实现。gate 检查： + +- validation 总分提升阈值； +- 训练提升但 validation 不提升的过拟合； +- 新 hard fail； +- protected case 退化； +- 单 case 最大降分； +- 可选成本预算； +- baseline 与 candidate case ID 集合一致。 + +候选缺失逐 case 数据、成本 gate 所需数据不完整或评测失败时，gate 必须拒绝并返回明确原因。 + +### `eval_loop/writeback.py` + +新增 prompt 快照和回写组件： + +- 保存每个 prompt 文件的原始字节与 SHA-256； +- 候选复评期间临时应用 prompt，并在 `finally` 中恢复； +- 恢复后校验哈希，恢复失败立即终止 pipeline； +- 最终回写前执行 compare-and-swap 检查，避免覆盖并发修改； +- 多 prompt 使用临时文件和 `os.replace`，任一失败时回滚所有已写字段； +- 返回结构化 `WritebackResult`。 + +### `eval_loop/report.py` + +只负责 schema 序列化、Markdown 渲染和 run-specific 审计目录。它不再推导 gate 或 SDK 特殊结果。 + +## 统一数据模型 + +`CaseResult` 增加逐 metric 数据和 trace 可用性： + +```python +@dataclass(frozen=True) +class CaseResult: + case_id: str + split: str + score: float + metrics: dict[str, float] + passed: bool + output: str + trace: dict[str, object] + trace_available: bool + failure_category: str | None + failure_reason: str | None + evidence: str | None + cost: float + hard_failed: bool + expected_failure_category: str | None +``` + +新增以下审计模型： + +- `OptimizationRound`：round ID、候选 prompt bundle、修改理由、optimizer 指标、成本和耗时； +- `CostSummary`：optimizer、evaluator、agent、total 和 `complete`； +- `WritebackResult`：`rejected` 表示没有候选通过 gate，`not_requested` 表示候选通过但用户未要求回写，`applied` 表示回写成功，`rolled_back` 表示写入失败且已完整恢复，`rollback_failed` 表示恢复不完整；同时记录前后哈希和错误原因； +- `OptimizationResult`：所有去重候选、round 记录和 backend 原始摘要。 + +## 端到端数据流 + +1. 解析并严格验证四类输入文件、gate 配置和 target prompt 路径。 +2. 生成唯一 `run_id`，创建 `runs/.tmp/`，启动 `perf_counter`。 +3. 对输入文件和源 prompt 做字节级快照与 SHA-256。 +4. 用 baseline prompt 分别评测 train 和 validation；任一 baseline 评测缺 case 或恢复 prompt 失败时终止。 +5. 仅根据 baseline train 失败结果生成失败归因和候选。 +6. 对每个候选分别完整评测 train 和 validation，校验 case ID 一致，再计算 delta。 +7. 对每个候选执行同一完整 gate，并从已接受候选中按 validation、train、原始顺序稳定选择最佳项。 +8. 写候选、round、评测结果、delta、gate、输入快照哈希和真实耗时到临时 run 目录。 +9. 若没有接受候选或未指定 `--update-source`，记录对应 writeback 状态。 +10. 若允许回写，先确认源文件仍与起始快照一致，再执行原子多文件写入；失败则回滚并记录原因。 +11. 完成 `writeback.json`、最终 JSON/Markdown 后，把临时目录原子重命名为 `runs//`。已存在的 run ID 不允许覆盖。 +12. 根目录 `optimization_report.json` 和 `.md` 仅作为最新结果的便利副本；不可变证据以 run 目录为准。 + +## Fake 模式隔离规则 + +fake model 只能读取用户输入和当前 prompt 文本。evalset 的 expectation、expected answer、split、protected 和标签只能由 evaluator、归因器或 gate 使用。 + +公开样例输入改成包含明确业务值的指令，例如 `intent=refund, priority=high`。fake model 从用户指令解析值：baseline 对严格格式请求加入多余说明，过度候选对所有请求强制 JSON，安全候选只在用户明确要求时使用严格格式。相同输入和 prompt 在 train/validation 中必须产生相同输出。 + +fake optimizer 根据 baseline train 中的失败类别选择规则模板。例如发现 format 与 exact-answer 失败时，生成一个全局严格候选和一个按请求限定的候选。它不得读取 validation 结果或 case ID。 + +## 成本语义 + +`max_total_cost` 是可选 gate。`CostSummary.complete=True` 时，total 必须是已知 optimizer、evaluator 和 agent 成本之和。fake backend 提供完整成本。 + +SDK 无法获得 agent/provider 完整成本时设置 `complete=False`。若配置了成本上限，候选必须以 `cost_unavailable` 拒绝；未配置成本上限时可以继续其他 gate，但报告必须把 SDK 提供的数值标记为 `reported_optimizer_cost`，不能称为完整总成本。 + +## 错误处理 + +- baseline 输入、评测或 prompt 恢复失败：终止运行，不选择候选，不最终回写。 +- 单个候选评测失败：记录候选拒绝原因，继续评测其他候选。 +- case ID 缺失、重复或集合不一致：该候选拒绝。 +- 非有限数值、非标准 JSON、重复 target path、split 冲突：输入阶段直接报错。 +- 最终回写 compare-and-swap 失败：不覆盖源文件，记录并抛出并发修改错误。 +- 多文件写入失败：回滚已写字段；回滚成功时状态为 `rolled_back`，回滚失败时状态为 `rollback_failed` 并报告受影响路径。 +- 审计写入失败：最终回写不会开始。 + +## 审计产物 + +每个 `runs//` 至少包含： + +- `optimization_report.json` 与 `optimization_report.md`； +- `input_hashes.json` 和规范化配置快照； +- `baseline_prompts/`、`candidate_prompts/` 与 `prompt_diffs/`； +- `case_results/` 和 `per_case_deltas.json`； +- `rounds/`，保存每轮候选、理由、成本、耗时和 optimizer 指标； +- `gate_decisions.json`； +- `writeback.json`； +- SDK 模式下的 `sdk_optimizer/` 原始产物。 + +报告路径优先保存仓库相对路径；仓库外输入仅保存用户传入的规范化路径，不写入维护者机器生成样例。示例报告由测试临时生成并与 committed inputs 的哈希自动比对。 + +## 测试设计 + +### Backend 契约测试 + +fake 与 SDK backend 都必须返回完整、case ID 唯一且集合一致的 `EvalResult`。SDK 测试使用真实 `AgentEvaluator`、`AgentOptimizer` 和 `TargetPrompt`；仅 monkeypatch GEPA/外部反思模型调用，沿用仓库现有 facade 测试模式。 + +### 安全回写测试 + +- gate 拒绝且传入 `--update-source` 时，所有源 prompt 字节和哈希保持不变； +- gate 接受时才写入选中候选； +- 多 prompt 第二个字段写入失败时，第一个字段回滚； +- 起始快照后发生并发修改时拒绝覆盖； +- 审计写入失败时不触发回写。 + +### 隐藏决策准确率 + +新增至少 10 个与公开六例分离的 holdout 场景，覆盖安全提升、无提升、训练提升而 validation 退化、protected regression、new hard fail、单 case drop 和超预算。标签与 pipeline 输入分离，计算 `correct / total` 并断言 `>= 0.80`。 + +### 失败归因准确率 + +新增独立归因语料，覆盖 format、final response、tool、parameter、rubric 和 knowledge 类别。归因器只接收评测错误和证据，不接收标签；断言准确率 `>= 0.75`，且每个失败都有非空 reason 与 evidence。 + +### 性能与报告 + +用 wall clock 运行完整 fake+trace pipeline，断言少于 180 秒；报告中的 duration 必须大于零且不大于测试观测值的合理上界。测试重新生成 example report，校验输入哈希、相对路径、严格 JSON、全部必需字段和旧 run 不被覆盖。 + +## 兼容性与迁移 + +- 保留现有 CLI 参数和同步 `run_pipeline()`；新增异步入口不破坏已有调用方。 +- fake 和 SDK 输入统一使用官方 SDK EvalSet 形状；loader 在本次 PR 内继续兼容旧 `cases` 形状，但 README 和 committed examples 只展示官方形状。 +- JSON schema 版本提升，新增字段不复用旧字段表达不同语义。 +- `--update-source` 的用户语义保持不变，但实际执行从 optimizer 内部提前写入改为完整 gate 后写入。 + +## 合并标准 + +以下条件全部满足后才建议合并： + +1. 六个公开 case 完整运行并生成 JSON、Markdown 和审计目录； +2. SDK 路径提供真实 baseline/candidate 逐 case 结果，不存在可被接受的 partial gate； +3. 训练提升而 validation 退化的候选被拒绝； +4. holdout 决策准确率至少 80%； +5. 独立归因准确率至少 75%，所有失败可解释； +6. fake+trace wall clock 少于三分钟且报告记录真实耗时； +7. gate 拒绝、评测失败、审计失败和并发修改时源 prompt 不被覆盖； +8. committed example report 的哈希与 committed inputs 一致且没有贡献者绝对路径； +9. 目标测试、完整示例测试、仓库 lint、build 和 CI 全部通过。 diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..ac11c3f7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,9 @@ +# 设计说明 + +本示例把基线评测、失败归因、提示词优化、候选复评、接受门禁和审计落盘串成同一条可复现流水线。训练集与验证集采用 SDK 标准 EvalSet；FakeModel 只读取提示词和用户输入，不能接触期望答案、标签、受保护标记或样例编号，真值仅供评测器、归因器和门禁使用，从源头避免答案泄漏。 + +失败结果按格式、最终回复、工具调用、参数、知识召回和规则评分等类别聚合，每个失败样例保留原因与证据。训练集只用于暴露问题，所有去重后的候选都必须重新执行完整训练集和验证集评测，再逐样例计算新增通过、新增失败、升分和降分。统一门禁检查验证集增益、过拟合、新增硬失败、受保护样例退化、单例降分和累计成本；总成本不完整时，任何数值预算都以 `cost_unavailable` 拒绝。 + +SDK 优化始终使用 `update_source=False`，并记录真实逐样例指标、失败证据及轨迹是否可用。只有候选通过完整门禁、输入完整性检查且预写审计成功后，`--update-source` 才由共享流水线事务性回写所选提示词；失败时保留原始字节或明确报告恢复错误。 + +运行产物先写入 `runs/..tmp/`，包含基线与候选提示词、每轮记录、全部分割评测结果、逐样例差异、门禁、成本和回写日志。发布时采用不可覆盖的原子目录重命名：Windows 使用 MoveFileExW，Linux 使用 renameat2；其他缺少安全原语的平台直接失败。最终目录不可变，清单记录每个文件的 SHA-256 与大小，顶层报告仅是发布后的便利副本。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..89992154 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,225 @@ +# Evaluation + Optimization Closed Loop + +This example implements issue #91 as a reproducible evaluation + optimization +loop. The default path is deterministic fake mode, so it runs in CI and on a +fresh checkout without `TRPC_AGENT_API_KEY` or any external model provider. A +real SDK adapter path is also present in `eval_loop/backends.py` for +`AgentOptimizer` / `TargetPrompt` integration. + +## Architecture + +```text +train.evalset.json + val.evalset.json + optimizer.json + baseline prompt + | + v +loader.py / config.py --> validated cases, gate config, input hashes + | + v +backends.py + |-- FakeBackend -> FakeModel + FakeJudge + FakeOptimizer + `-- SDKBackend -> AgentOptimizer + TargetPrompt + user call_agent + AgentEvaluator + | + v +attribution.py -> evaluator.py -> gate.py -> report.py + | + v +optimization_report.json / optimization_report.md / runs// audit files +``` + +## Quick Start + +One-command fake mode: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --fake-model --fake-judge --trace +``` + +Equivalent new form: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --trace +``` + +Full fake command: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --train examples/optimization/eval_optimize_loop/data/train.evalset.json \ + --val examples/optimization/eval_optimize_loop/data/val.evalset.json \ + --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json \ + --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt \ + --output-dir /tmp/eval-optimize-loop \ + --mode fake \ + --trace +``` + +SDK adapter command shape: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode sdk \ + --train path/to/sdk_train.evalset.json \ + --val path/to/sdk_val.evalset.json \ + --optimizer-config path/to/sdk_optimizer.json \ + --prompt path/to/system_prompt.txt \ + --sdk-call-agent your_package.your_module:call_agent \ + --output-dir /tmp/eval-optimize-loop-sdk +``` + +Optional wrapper gate and multi-prompt form: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode sdk \ + --train path/to/sdk_train.evalset.json \ + --val path/to/sdk_val.evalset.json \ + --optimizer-config path/to/sdk_optimizer.json \ + --gate-config path/to/wrapper_gate.json \ + --target-prompt system_prompt=prompts/system.md \ + --target-prompt router_prompt=prompts/router.md \ + --sdk-call-agent your_package.your_module:call_agent \ + --run-id local-sdk-smoke \ + --output-dir /tmp/eval-optimize-loop-sdk +``` + +`--sdk-call-agent` must point to an async callable compatible with +`AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials +needed by that callable. SDK mode never silently falls back to fake mode. The +generated reproducibility command records the actual `--sdk-call-agent` +`module:function` target and file/config paths, but it does not record API keys +or other provider secrets. Persisted optimizer config snapshots recursively +replace credential values with ``. + +The checked-in `train.evalset.json` and `val.evalset.json` use the SDK +`EvalSet` schema (`eval_set_id` / `eval_cases`). Fake mode reads its +deterministic judge metadata from each case's `session_input.state` +(`eval_optimize_expectation`, `eval_optimize_tags`, and +`eval_optimize_protected`). + +SDK optimizer config and wrapper gate config are intentionally separate. +`--optimizer-config` is passed unchanged to `AgentOptimizer.optimize`, so it must +follow the SDK `OptimizeConfigFile` schema. Put wrapper-only gate settings in +`--gate-config` (for example `{"gate": {"min_val_score_improvement": 0.05, +"allow_new_hard_fail": false, "max_score_drop_per_case": 0.0, +"max_total_cost": null}}`). If `--gate-config` is omitted, the wrapper uses the +same default gate values as fake mode and auto-adds validation cases marked with +`eval_optimize_protected`. A numeric `max_total_cost` fails closed with +`cost_unavailable` whenever the backend cannot prove a complete total. Use +`null` only when you intentionally want to run the quality gates without a cost +limit; `reported_optimizer_cost` remains an incomplete lower bound, not a total. + +`--target-prompt name=path` may be repeated. If omitted, SDK mode keeps the old +single-field behavior and registers `system_prompt=--prompt`. A run can optimize +only `router_prompt`, only `skill_prompt`, or any set of named fields as long as +`OptimizeResult.best_prompts` returns every registered field. + +Fake mode is the deterministic offline closed loop. SDK mode is the real +`AgentOptimizer` / `TargetPrompt` path followed by explicit post-optimization +`AgentEvaluator` runs. The adapter deduplicates every prompt bundle found in +`OptimizeResult.rounds[].candidate_prompts` and `best_prompts`. The shared +pipeline temporarily installs each bundle, reruns the complete train and +validation evalsets, restores the exact source bytes, then computes the same +per-case deltas and complete gate decision used by fake mode. SDK case results +carry real per-metric scores, failure reason/evidence, and `trace_available`; +missing SDK trace data is reported as unavailable rather than fabricated. +Optimizer aggregates remain audit context and are never substituted for these +post-optimization case results. + +Both modes default to a collision-resistant run id of the form +`eval_optimize_loop___`. Pass `--run-id` +when a stable path is useful. Existing temporary or final run ids are never +reused or overwritten. The checked-in example is generated in a fresh output +directory with the explicit id `example`. + +## Source Prompt Writes + +The default is **no source write-back**. `SDKBackend` always calls +`AgentOptimizer.optimize(update_source=False)` so the optimizer can never commit +early. `--update-source` is handled only by the shared pipeline: after all +candidate evaluations, full gate acceptance, input-integrity checks, and audit +preparation, it transactionally writes the selected prompt bundle. This rule is +the same in fake and SDK modes. The report records the requested action and the +terminal `writeback` state. + +## Candidate Behavior + +The fake optimizer proposes exactly two candidates: + +- `candidate_001_overfit`: fixes train formatting but forces JSON too broadly; + it improves train score and regresses validation, so the gate rejects it. +- `candidate_002_safe`: applies strict JSON/exact-answer behavior only when + requested; it improves validation without protected-case regression, so the + gate may accept it. + +The fake model reads only the prompt and user input. It cannot inspect +expectations, labels, tags, protected markers, or case ids. Those fields are +available only to the fake judge, failure attribution, and acceptance gate. + +## Reports + +`optimization_report.json` uses `eval_optimize_loop.v2` and includes: + +- `schema_version`; +- `run` metadata: mode, fake flags, trace flag, case counts, update-source flag, + and input paths; +- `baseline` plus compatibility fields `baseline_train` and + `baseline_validation`; +- `baseline_prompts`, all candidate prompt bundles, complete train/validation + results, rationale, and prompt diff; +- optimizer `rounds`, `cost_summary`, and terminal `writeback`; +- per-case deltas with `delta_type` (`new_pass`, `new_fail`, `score_up`, + `score_down`, `unchanged`); +- failure attribution summary and attribution accuracy when expected labels are + present; +- gate decisions with overfit detection, protected regressions, new hard + failures, excessive drops, and cost fields in both fake and SDK modes; +- audit data: seed, real duration, redacted config and raw-file hashes, input + hashes, candidate artifact mapping, complete/incomplete cost semantics, + prompt integrity journal, and reproducibility command. + +When `CostSummary.complete` is true, `gate.max_total_cost` is interpreted as the +total evaluated run cost when each candidate is judged: baseline cost plus all +candidates evaluated so far, including rejected candidates. When it is false, +the report separates `known_run_cost` and `reported_optimizer_cost`, and any +numeric total-cost gate rejects with `cost_unavailable`. + +`optimization_report.md` includes final decision, gate reasons, score table, +per-case delta table, failure attribution summary, cost/audit details, prompt +diffs, and the reproducibility command. + +Audit data is first written under `output_dir/runs/..tmp/`. After every +final artifact and prompt-integrity check succeeds, the directory is published +once to `output_dir/runs//` with no-replace semantics. Windows uses +`MoveFileExW` without a replace flag; Linux uses +`renameat2(RENAME_NOREPLACE)`. A POSIX platform without that atomic primitive +fails closed. Published run directories are authoritative and immutable; the +top-level JSON/Markdown files are convenience copies created afterward. + +Each run contains `pre_write_report.*`, final reports, a redacted config +snapshot, input hashes, `baseline_prompts/`, every candidate prompt field and +diff, baseline and candidate split results, per-round JSON, per-case deltas, +gate decisions, evaluation failures, `writeback.json`, and the durable +writeback journal. SDK optimizer artifacts are kept beneath `optimizer/` with a +redacted config snapshot. `artifact_manifest.json` maps the logical artifacts +and records the SHA-256 and byte size of every file. + +The repository keeps only stable examples: + +- `outputs/optimization_report.example.json` +- `outputs/optimization_report.example.md` + +Runtime `optimization_report.json`, `optimization_report.md`, and `runs/` +directories are not committed. + +## Run Tests + +```bash +python -m pytest examples/optimization/eval_optimize_loop/tests +``` + +The tests cover fake hidden-sample generalization, SDK EvalSet compatibility, +config validation, gate rejection paths, protected-case behavior, failure +attribution, tool/knowledge judge paths, SDK adapter and post-evaluator wiring +through monkeypatching, deterministic report generation, and both CLI forms. CI +can run fake mode plus the monkeypatched SDK smoke tests without real API +credentials; real SDK/model calls are opt-in local or integration runs. diff --git a/examples/optimization/eval_optimize_loop/data/optimizer.json b/examples/optimization/eval_optimize_loop/data/optimizer.json new file mode 100644 index 00000000..5886597c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/optimizer.json @@ -0,0 +1,48 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 91, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 2048, + "temperature": 0.4 + } + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "frontier_type": "instance", + "reflection_minibatch_size": 3, + "reflection_history_top_k": 2, + "skip_perfect_score": false, + "use_merge": false, + "max_metric_calls": 18, + "score_threshold": 1.0, + "max_iterations_without_improvement": 3 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 00000000..9b162057 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,144 @@ +{ + "eval_set_id": "eval_optimize_loop_train_v1", + "name": "Evaluation optimization loop train set", + "description": "Training cases used for failure attribution and prompt optimization.", + "eval_cases": [ + { + "eval_id": "train_json_refund", + "conversation": [ + { + "invocation_id": "train_json_refund_turn_1", + "user_content": { + "parts": [ + { + "text": "Return strict JSON with intent=refund and priority=high." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"intent\":\"refund\",\"priority\":\"high\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "trainer", + "state": { + "eval_optimize_expectation": { + "type": "json", + "required_keys": [ + "intent", + "priority" + ], + "expected_values": { + "intent": "refund", + "priority": "high" + }, + "expected_failure_category": "format_violation" + }, + "eval_optimize_tags": [ + "json", + "format" + ] + } + } + }, + { + "eval_id": "train_exact_order_status", + "conversation": [ + { + "invocation_id": "train_json_order_turn_1", + "user_content": { + "parts": [ + { + "text": "Return strict JSON with status=READY and next_step=ship." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"next_step\":\"ship\",\"status\":\"READY\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "trainer", + "state": { + "eval_optimize_expectation": { + "type": "json", + "required_keys": [ + "status", + "next_step" + ], + "expected_values": { + "status": "READY", + "next_step": "ship" + }, + "expected_failure_category": "format_violation" + }, + "eval_optimize_tags": [ + "json", + "format" + ] + } + } + }, + { + "eval_id": "train_rubric_retry_summary", + "conversation": [ + { + "invocation_id": "train_latency_retries_turn_1", + "user_content": { + "parts": [ + { + "text": "Explain latency and retries naturally in under 80 characters." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "Latency can trigger retries." + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "trainer", + "state": { + "eval_optimize_expectation": { + "type": "rubric", + "must_include": [ + "latency", + "retries" + ], + "forbidden": [ + "database" + ], + "max_chars": 80 + }, + "eval_optimize_tags": [ + "rubric", + "no_effect" + ] + } + } + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 00000000..9262b14f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,141 @@ +{ + "eval_set_id": "eval_optimize_loop_validation_v1", + "name": "Evaluation optimization loop validation set", + "description": "Validation cases that separate safe optimization from train-set overfitting.", + "eval_cases": [ + { + "eval_id": "val_json_invoice", + "conversation": [ + { + "invocation_id": "val_json_approval_turn_1", + "user_content": { + "parts": [ + { + "text": "Return strict JSON with status=approved and next_step=email_customer." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"next_step\":\"email_customer\",\"status\":\"approved\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "validator", + "state": { + "eval_optimize_expectation": { + "type": "json", + "required_keys": [ + "status", + "next_step" + ], + "expected_values": { + "next_step": "email_customer", + "status": "approved" + }, + "expected_failure_category": "format_violation" + }, + "eval_optimize_tags": [ + "json", + "safe_improves" + ] + } + } + }, + { + "eval_id": "val_explain_cache", + "conversation": [ + { + "invocation_id": "val_cache_stale_turn_1", + "user_content": { + "parts": [ + { + "text": "Explain cache invalidation and stale data naturally." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "Cache invalidation decides when cached data is stale data and must be refreshed." + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "validator", + "state": { + "eval_optimize_expectation": { + "type": "rubric", + "must_include": [ + "cache", + "stale data" + ], + "forbidden": [ + "{", + "}", + "json" + ], + "max_chars": 180, + "expected_failure_category": "format_violation" + }, + "eval_optimize_tags": [ + "rubric", + "overfit_guard" + ] + } + } + }, + { + "eval_id": "val_protected_yes_no", + "conversation": [ + { + "invocation_id": "val_return_yes_turn_1", + "user_content": { + "parts": [ + { + "text": "Return only YES; do not use JSON." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "YES" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "validator", + "state": { + "eval_optimize_expectation": { + "type": "exact", + "expected": "YES", + "expected_failure_category": "final_response_mismatch" + }, + "eval_optimize_tags": [ + "protected", + "exact" + ], + "eval_optimize_protected": true + } + } + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/eval_loop/__init__.py b/examples/optimization/eval_optimize_loop/eval_loop/__init__.py new file mode 100644 index 00000000..74835adb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/__init__.py @@ -0,0 +1,24 @@ +"""Deterministic evaluation + optimization loop example. + +This package is intentionally example-local. It mirrors the shape of an +Evaluator/Optimizer workflow while keeping fake model and fake judge execution +offline, deterministic, and easy to inspect. +""" + +from .schemas import CandidatePrompt +from .schemas import CaseDelta +from .schemas import CaseResult +from .schemas import EvalCase +from .schemas import EvalResult +from .schemas import GateDecision +from .schemas import OptimizationReport + +__all__ = [ + "CandidatePrompt", + "CaseDelta", + "CaseResult", + "EvalCase", + "EvalResult", + "GateDecision", + "OptimizationReport", +] diff --git a/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py b/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py new file mode 100644 index 00000000..11e02b45 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/artifacts.py @@ -0,0 +1,86 @@ +"""Cross-platform validation for values used as artifact path components.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +ARTIFACT_COMPONENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +WINDOWS_RESERVED_NAMES = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{index}" for index in range(1, 10)), + *(f"LPT{index}" for index in range(1, 10)), +} +MAX_ARTIFACT_COMPONENT_LENGTH = 128 + + +def validate_artifact_component(value: Any, *, context: str) -> str: + """Return a portable artifact component or fail before filesystem access.""" + + if (not isinstance(value, str) or value in {"", ".", ".."} or len(value) > MAX_ARTIFACT_COMPONENT_LENGTH + or value.endswith((".", " ")) or not ARTIFACT_COMPONENT_RE.fullmatch(value)): + raise ValueError(f"unsafe {context}: {value!r}") + windows_stem = value.split(".", 1)[0].upper() + if windows_stem in WINDOWS_RESERVED_NAMES: + raise ValueError(f"unsafe {context}: {value!r} is reserved on Windows") + return value + + +def validate_distinct_file_paths( + paths: Mapping[str, str | Path], + *, + context: str, +) -> None: + """Reject aliases using portable case-folding and physical file identity.""" + + resolved_keys: dict[str, str] = {} + physical_keys: dict[tuple[int, int], str] = {} + observed_paths: list[tuple[str, Path, bool]] = [] + for label, raw_path in paths.items(): + path = Path(raw_path) + try: + resolved = path.resolve(strict=False) + except OSError as error: + raise ValueError(f"{context} {label!r} cannot be resolved: {error}") from error + + portable_key = str(resolved).casefold() + previous_label = resolved_keys.get(portable_key) + if previous_label is not None: + raise ValueError(f"{context} must be different physical files; " + f"{previous_label!r} and {label!r} collide case-insensitively") + + try: + metadata = path.stat() + except (FileNotFoundError, NotADirectoryError): + metadata = None + except OSError as error: + raise ValueError(f"{context} {label!r} is unavailable: {error}") from error + + physical_key = ((int(metadata.st_dev), + int(metadata.st_ino)) if metadata is not None and int(metadata.st_ino) != 0 else None) + if physical_key is not None and physical_key in physical_keys: + raise ValueError(f"{context} must be different physical files; " + f"{physical_keys[physical_key]!r} and {label!r} are aliases") + + for observed_label, observed_path, observed_exists in observed_paths: + if metadata is None or not observed_exists: + continue + try: + same_file = path.samefile(observed_path) + except (FileNotFoundError, NotADirectoryError): + same_file = False + except OSError as error: + raise ValueError(f"{context} {label!r} is unavailable: {error}") from error + if same_file: + raise ValueError(f"{context} must be different physical files; " + f"{observed_label!r} and {label!r} are aliases") + + resolved_keys[portable_key] = label + if physical_key is not None: + physical_keys[physical_key] = label + observed_paths.append((label, path, metadata is not None)) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/attribution.py b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py new file mode 100644 index 00000000..2b298bc8 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py @@ -0,0 +1,80 @@ +"""Rule-based failure attribution for the example evaluator.""" + +from __future__ import annotations + +from collections import Counter +from typing import Iterable + +from .schemas import EvalResult + +_ERROR_TO_ATTRIBUTION = { + "json_parse_failure": ("format_violation", "output is not valid JSON"), + "required_key_missing": ("final_response_mismatch", "required JSON key is missing"), + "json_value_mismatch": ("final_response_mismatch", "JSON value does not match expected value"), + "exact_answer_mismatch": ("final_response_mismatch", "normalized exact answer mismatch"), + "forbidden_pattern": ("format_violation", "output contains a forbidden pattern"), + "missing_rubric_terms": ("llm_rubric_not_met", "required rubric terms are missing"), + "max_chars_exceeded": ("length_violation", "output exceeds max_chars"), + "tool_call_error": ("tool_call_error", "tool call did not match expected tool"), + "parameter_error": ("parameter_error", "tool call parameters did not match"), + "knowledge_recall_insufficient": ( + "knowledge_recall_insufficient", + "required knowledge evidence was not recalled", + ), +} + + +def attribute_failure(error_code: str, evidence: str) -> tuple[str, str, str]: + """Return failure_category, failure_reason, evidence for a judge error.""" + + category, reason = _ERROR_TO_ATTRIBUTION.get( + error_code, + ("unknown_failure", f"unmapped judge error: {error_code}"), + ) + return category, reason, evidence + + +def summarize_failures(results: Iterable[EvalResult]) -> dict[str, object]: + """Summarize failures by category and by prompt/split for reporting.""" + + by_category: Counter[str] = Counter() + by_prompt: dict[str, dict[str, int]] = {} + examples: list[dict[str, str]] = [] + total_failed = 0 + expected_total = 0 + expected_correct = 0 + + for result in results: + prompt_key = f"{result.prompt_id}:{result.split}" + by_prompt.setdefault(prompt_key, {}) + for case in result.cases: + if case.passed: + continue + total_failed += 1 + category = case.failure_category or "unknown_failure" + by_category[category] += 1 + by_prompt[prompt_key][category] = by_prompt[prompt_key].get(category, 0) + 1 + if case.expected_failure_category: + expected_total += 1 + if case.expected_failure_category == category: + expected_correct += 1 + examples.append({ + "prompt_id": result.prompt_id, + "split": result.split, + "case_id": case.case_id, + "failure_category": category, + "failure_reason": case.failure_reason or "", + "evidence": case.evidence or "", + }) + + return { + "total_failed_cases": total_failed, + "by_category": dict(sorted(by_category.items())), + "by_prompt_split": { + key: dict(sorted(value.items())) + for key, value in sorted(by_prompt.items()) + }, + "examples": examples, + "attribution_accuracy": (round(expected_correct / expected_total, 6) if expected_total else None), + "expected_labeled_failures": expected_total, + } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py new file mode 100644 index 00000000..e8ad751b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -0,0 +1,1248 @@ +"""Backend-neutral adapters for fake and SDK optimization paths.""" + +from __future__ import annotations + +import asyncio +import importlib +import math +import time +from collections.abc import Iterable +from dataclasses import dataclass +from dataclasses import replace +from pathlib import Path +from typing import Any +from typing import Protocol + +from .diffing import make_unified_diff +from .artifacts import validate_distinct_file_paths +from .evaluator import ExampleEvaluator +from .fake_judge import FakeJudge +from .fake_model import FakeModel +from .loader import load_eval_cases +from .loader import read_json +from .optimizer import FakeOptimizer +from .schemas import CandidatePrompt +from .schemas import CaseResult +from .schemas import CostSummary +from .schemas import EvalCase +from .schemas import EvalResult +from .schemas import OptimizationResult +from .schemas import OptimizationRound +from .writeback import snapshot_prompt_files +from .writeback import temporary_prompt_bundle + + +class EvaluationBackend(Protocol): + """Common asynchronous evaluation contract.""" + + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + ... + + +class OptimizationBackend(Protocol): + """Common asynchronous optimization contract.""" + + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + ... + + +@dataclass +class FakeBackend: + """Deterministic backend used by the self-contained example.""" + + seed: int = 91 + trace_enabled: bool = False + + def __post_init__(self) -> None: + self._optimizer = FakeOptimizer() + + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + del artifact_dir + prompt = _required_system_prompt(prompts, context=f"cannot evaluate {prompt_id}") + cases = _load_sdk_expected_cases(dataset_path, split=split).values() + evaluator = ExampleEvaluator( + FakeModel(seed=self.seed), + FakeJudge(), + trace_enabled=trace, + ) + result = evaluator.evaluate( + prompt_id=prompt_id, + prompt=prompt, + cases=cases, + split=split, + ) + if not trace: + return result + + sanitized_cases = [] + for case_result in result.cases: + model_trace = case_result.trace.get("model") + sanitized_trace = dict(model_trace) if isinstance(model_trace, dict) else {} + sanitized_cases.append(replace( + case_result, + trace=sanitized_trace, + trace_available=bool(sanitized_trace), + )) + return replace(result, cases=sanitized_cases) + + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + del train_path, validation_path, config_path, artifact_dir + baseline_prompt = _required_system_prompt( + baseline_prompts, + context="cannot optimize fake prompt bundle", + ) + proposal_started_at = time.perf_counter() + candidates = _normalize_fake_candidates( + self._optimizer.propose( + baseline_prompt, + baseline_train, + failure_summary, + )) + proposal_duration_seconds = _positive_perf_duration( + proposal_started_at, + time.perf_counter(), + ) + round_duration_seconds = (proposal_duration_seconds / len(candidates) if candidates else 0.0) + zero_cost = CostSummary(complete=True) + rounds = [ + OptimizationRound( + round_id=index, + candidate_id=candidate.candidate_id, + prompts=candidate.bundle(), + rationale=candidate.rationale, + metrics={}, + cost=zero_cost, + duration_seconds=round_duration_seconds, + ) for index, candidate in enumerate(candidates, start=1) + ] + return OptimizationResult( + candidates=candidates, + rounds=rounds, + cost=zero_cost, + raw_summary={ + "backend": "fake", + "baseline_prompt_id": baseline_train.prompt_id, + "failure_summary": _safe_jsonable(failure_summary), + "proposal_duration_seconds": proposal_duration_seconds, + "round_duration_allocation": ("equal_share_of_batch_proposal_duration"), + }, + ) + + def optimize( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + """Reject legacy optimization that has no observed failure evidence.""" + + del baseline_prompt, train_path, val_path, optimizer_config_path, output_dir + raise RuntimeError("FakeBackend.optimize() cannot invent training failure evidence; " + "use await FakeBackend.optimize_candidates(...) with baseline_train " + "and failure_summary.") + + +class SDKBackend: + """Adapter around SDK AgentEvaluator, AgentOptimizer, and TargetPrompt. + + update_source is accepted only through legacy keyword forwarding so the + current synchronous pipeline can transition independently. It is never + stored or delegated; source writeback belongs to the wrapper after gating. + """ + + def __init__( + self, + prompt_path: str | Path, + call_agent_path: str | None = None, + target_prompt_paths: dict[str, str | Path] | None = None, + **legacy_options: object, + ) -> None: + unexpected = sorted(set(legacy_options) - {"update_source"}) + if unexpected: + raise TypeError(f"unexpected SDKBackend options: {', '.join(unexpected)}") + self.prompt_path = prompt_path + self.call_agent_path = call_agent_path + self.target_prompt_paths = dict(target_prompt_paths) if target_prompt_paths else None + self.last_result: Any | None = None + self.last_result_summary: dict[str, Any] | None = None + self.last_artifact_dir: str | None = None + self.last_baseline_prompts: dict[str, str] | None = None + self.last_best_prompts: dict[str, str] | None = None + + def optimize( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + """Safely bridge old synchronous callers to the async implementation.""" + + if _has_running_loop(): + raise ValueError("SDKBackend.optimize() cannot be called while an event loop is already running; " + "use await SDKBackend.optimize_async(...) instead.") + return asyncio.run( + self.optimize_async( + baseline_prompt=baseline_prompt, + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + output_dir=output_dir, + )) + + async def optimize_async( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + """Compatibility async wrapper returning the historical candidate list.""" + + target_paths = self._target_prompt_paths() + try: + baseline_prompts = _read_prompt_bundle(target_paths) + except FileNotFoundError: + # Let optimize_candidates report dependency/import failures before + # it reaches its authoritative source snapshot. + baseline_prompts = {name: baseline_prompt for name in target_paths} + result = await self.optimize_candidates( + baseline_prompts=baseline_prompts, + baseline_train=EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ), + failure_summary={}, + train_path=train_path, + validation_path=val_path, + config_path=optimizer_config_path, + artifact_dir=output_dir, + ) + return result.candidates + + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + del baseline_train, failure_summary + call_agent = self._load_required_call_agent(for_evaluation=False) + try: + from trpc_agent_sdk.evaluation import AgentOptimizer + from trpc_agent_sdk.evaluation import TargetPrompt + except Exception as exc: # pragma: no cover - depends on optional SDK import health + raise ValueError(f"sdk mode could not import AgentOptimizer/TargetPrompt: {exc}") from exc + + target_paths = self._target_prompt_paths() + snapshot = snapshot_prompt_files(target_paths) + source_bundle = _prompt_bundle_from_snapshot(snapshot) + baseline_bundle = _validated_prompt_bundle( + baseline_prompts, + target_paths, + context="baseline prompt bundle", + ) + mismatched = sorted(name for name in target_paths if baseline_bundle[name] != source_bundle[name]) + if mismatched: + fields = ", ".join(mismatched) + raise ValueError(f"baseline prompt bundle does not match registered source prompt files: {fields}") + + target_prompt = TargetPrompt() + for name, path in target_paths.items(): + target_prompt.add_path(name, str(path)) + + try: + sdk_result = await AgentOptimizer.optimize( + config_path=str(config_path), + call_agent=call_agent, + target_prompt=target_prompt, + train_dataset_path=str(train_path), + validation_dataset_path=str(validation_path), + output_dir=str(artifact_dir), + update_source=False, + verbose=0, + ) + finally: + changed_sources = _changed_snapshot_files(snapshot) + if changed_sources: + fields = ", ".join(changed_sources) + raise RuntimeError(f"AgentOptimizer modified source prompt files despite update_source=False: {fields}") + + _require_successful_optimize_result(sdk_result) + total_llm_cost = _nonnegative_result_field( + "total_llm_cost", + getattr(sdk_result, "total_llm_cost", 0.0), + ) + _pass_rate_result_field( + "baseline_pass_rate", + getattr(sdk_result, "baseline_pass_rate", 0.0), + ) + _pass_rate_result_field( + "best_pass_rate", + getattr(sdk_result, "best_pass_rate", 0.0), + ) + _finite_result_field( + "pass_rate_improvement", + getattr(sdk_result, "pass_rate_improvement", 0.0), + ) + _nonnegative_result_field( + "duration_seconds", + getattr(sdk_result, "duration_seconds", 0.0), + ) + best_raw = getattr(sdk_result, "best_prompts", None) + if not best_raw: + raise ValueError("sdk mode completed but OptimizeResult.best_prompts was empty") + best_prompts = _validated_prompt_bundle( + best_raw, + target_paths, + context="OptimizeResult.best_prompts", + ) + + candidates: list[CandidatePrompt] = [] + rounds: list[OptimizationRound] = [] + seen_bundles: set[tuple[tuple[str, str], ...]] = set() + seen_round_ids: set[int] = set() + for index, sdk_round in enumerate(getattr(sdk_result, "rounds", []) or [], start=1): + round_id = _round_id(sdk_round, fallback=index) + if round_id in seen_round_ids: + raise ValueError(f"duplicate SDK round id: {round_id}") + seen_round_ids.add(round_id) + candidate_id = f"sdk_round_{round_id:03d}" + round_raw_prompts = getattr(sdk_round, "candidate_prompts", {}) or {} + round_prompts = (_validated_prompt_bundle( + round_raw_prompts, + target_paths, + context=f"SDK round {round_id} candidate_prompts", + ) if round_raw_prompts else {}) + rationale = str(getattr(sdk_round, "acceptance_reason", "") or "") + round_metrics = _finite_metric_map( + getattr(sdk_round, "metric_breakdown", {}) or {}, + context=f"SDK round {round_id} metric_breakdown", + ) + _pass_rate_number( + getattr(sdk_round, "train_pass_rate", 0.0), + context=f"SDK round {round_id} train_pass_rate", + ) + _pass_rate_number( + getattr(sdk_round, "validation_pass_rate", 0.0), + context=f"SDK round {round_id} validation_pass_rate", + ) + round_cost_value = _nonnegative_number( + getattr(sdk_round, "round_llm_cost", 0.0), + context=f"SDK round {round_id} round_llm_cost", + ) + duration_seconds = _nonnegative_number( + getattr(sdk_round, "duration_seconds", 0.0), + context=f"SDK round {round_id} duration_seconds", + ) + rounds.append( + OptimizationRound( + round_id=round_id, + candidate_id=candidate_id, + prompts=round_prompts, + rationale=rationale, + metrics=round_metrics, + cost=CostSummary( + optimizer=round_cost_value, + total=round_cost_value, + complete=False, + ), + duration_seconds=duration_seconds, + )) + if round_prompts: + bundle_key = _prompt_bundle_key(round_prompts) + if bundle_key not in seen_bundles: + seen_bundles.add(bundle_key) + candidates.append( + _candidate_from_bundle( + candidate_id=candidate_id, + prompts=round_prompts, + rationale=rationale, + baseline_prompts=baseline_bundle, + )) + + best_key = _prompt_bundle_key(best_prompts) + if best_key not in seen_bundles: + candidates.append( + _candidate_from_bundle( + candidate_id="sdk_best", + prompts=best_prompts, + rationale="Best prompt returned by AgentOptimizer.optimize.", + baseline_prompts=baseline_bundle, + )) + + raw_summary = _summarize_sdk_result(sdk_result) + cost = CostSummary( + optimizer=total_llm_cost, + total=total_llm_cost, + complete=False, + reported_optimizer_cost=total_llm_cost, + ) + result = OptimizationResult( + candidates=candidates, + rounds=rounds, + cost=cost, + raw_summary=raw_summary, + ) + self.last_result = sdk_result + self.last_result_summary = raw_summary + self.last_artifact_dir = str(artifact_dir) + self.last_baseline_prompts = baseline_bundle + self.last_best_prompts = best_prompts + return result + + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + del trace + call_agent = self._load_required_call_agent(for_evaluation=True) + try: + from trpc_agent_sdk.evaluation import AgentEvaluator + from trpc_agent_sdk.evaluation import EvalConfig + from trpc_agent_sdk.evaluation import EvaluationCasesFailed + except Exception as exc: # pragma: no cover - depends on optional SDK import health + raise ValueError(f"sdk mode could not import AgentEvaluator: {exc}") from exc + + target_paths = self._target_prompt_paths() + candidate_prompts = _validated_prompt_bundle( + prompts, + target_paths, + context=f"cannot evaluate {prompt_id}", + ) + expected_cases = _load_sdk_expected_cases(dataset_path, split=split) + artifact_path = Path(artifact_dir) + artifact_path.mkdir(parents=True, exist_ok=True) + eval_config_path = artifact_path / "eval_config.json" + eval_config_path.write_text( + EvalConfig(criteria={ + "final_response_avg_score": 1.0 + }).model_dump_json(indent=2), + encoding="utf-8", + ) + snapshot = snapshot_prompt_files(target_paths) + result: Any | None = None + with temporary_prompt_bundle(snapshot, candidate_prompts): + executer = AgentEvaluator.get_executer( + str(dataset_path), + call_agent=call_agent, + print_detailed_results=False, + print_summary_report=False, + eval_result_output_dir=str(artifact_dir), + eval_metrics_file_path_or_dir=str(eval_config_path), + ) + try: + await executer.evaluate() + except EvaluationCasesFailed: + result = executer.get_result() + if result is None: + raise + else: + result = executer.get_result() + + if result is None: + raise ValueError(f"AgentEvaluator returned no result for {dataset_path}") + return _eval_result_from_sdk_result( + result, + prompt_id=prompt_id, + split=split, + expected_cases=expected_cases.values(), + ) + + def _load_required_call_agent(self, *, for_evaluation: bool): + if not self.call_agent_path: + suffix = " for AgentEvaluator runs" if for_evaluation else ( + ". The callable must be async and compatible with " + "AgentOptimizer.optimize(call_agent=...). Also configure real model credentials required " + "by that callable, such as TRPC_AGENT_API_KEY/TRPC_AGENT_BASE_URL/TRPC_AGENT_MODEL_NAME.") + raise ValueError(f"sdk mode requires --sdk-call-agent module:function{suffix}") + return _load_call_agent(self.call_agent_path) + + def _target_prompt_paths(self) -> dict[str, str | Path]: + paths = (dict(self.target_prompt_paths) if self.target_prompt_paths else {"system_prompt": self.prompt_path}) + validate_distinct_file_paths(paths, context="SDK target prompt fields") + return paths + + +def _required_system_prompt(prompts: dict[str, str], *, context: str) -> str: + if "system_prompt" not in prompts: + raise ValueError(f"{context}: missing required prompt field 'system_prompt'") + prompt = prompts["system_prompt"] + if not isinstance(prompt, str) or not prompt.strip(): + raise ValueError(f"{context}: prompt field 'system_prompt' must be a non-empty string") + return prompt + + +def _normalize_fake_candidates(candidates: Iterable[CandidatePrompt]) -> list[CandidatePrompt]: + normalized: list[CandidatePrompt] = [] + for candidate in candidates: + bundle = candidate.bundle() + normalized.append( + CandidatePrompt( + candidate_id=candidate.candidate_id, + prompt=candidate.prompt, + rationale=candidate.rationale, + prompt_diff=candidate.prompt_diff, + prompt_fields=bundle, + )) + return normalized + + +def _load_call_agent(path: str): + if ":" not in path: + raise ValueError("--sdk-call-agent must use module:function format") + module_name, function_name = path.split(":", 1) + try: + module = importlib.import_module(module_name) + except Exception as exc: + raise ValueError(f"--sdk-call-agent target {path!r} could not import module {module_name!r}: {exc}") from exc + call_agent = getattr(module, function_name, None) + if call_agent is None: + raise ValueError(f"--sdk-call-agent target {path!r} was not found") + if not callable(call_agent): + raise ValueError(f"--sdk-call-agent target {path!r} was found but is not callable") + return call_agent + + +def _has_running_loop() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +def _validated_prompt_bundle( + prompts: Any, + paths: dict[str, str | Path], + *, + context: str, +) -> dict[str, str]: + if not isinstance(prompts, dict): + raise ValueError(f"{context} must be a prompt mapping") + missing_fields = sorted(name for name in paths if name not in prompts) + if missing_fields: + if context == "OptimizeResult.best_prompts": + fields = ", ".join(missing_fields) + raise ValueError("sdk mode completed but OptimizeResult.best_prompts is missing " + f"registered target fields: {fields}") + raise ValueError(f"{context} is missing registered target fields: {', '.join(missing_fields)}") + extra_fields = sorted(name for name in prompts if name not in paths) + if extra_fields: + raise ValueError(f"{context} contains unregistered target fields: {', '.join(extra_fields)}") + empty_fields = sorted(name for name in paths if not isinstance(prompts[name], str) or not prompts[name].strip()) + if empty_fields: + if context == "OptimizeResult.best_prompts": + fields = ", ".join(empty_fields) + raise ValueError("sdk mode completed but OptimizeResult.best_prompts contained " + f"empty registered target fields: {fields}") + raise ValueError(f"{context} contains empty registered target fields: {', '.join(empty_fields)}") + return {name: prompts[name] for name in paths} + + +def _read_prompt_bundle(paths: dict[str, str | Path]) -> dict[str, str]: + prompts: dict[str, str] = {} + for name, path in paths.items(): + prompt_path = Path(path) + try: + prompts[name] = prompt_path.read_bytes().decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError(f"source prompt field {name!r} is not valid UTF-8: {prompt_path}") from exc + return prompts + + +def _prompt_bundle_from_snapshot(snapshot: Any) -> dict[str, str]: + prompts: dict[str, str] = {} + for name, prompt_file in snapshot.files.items(): + try: + prompts[name] = prompt_file.content.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError(f"source prompt field {name!r} is not valid UTF-8: {prompt_file.path}") from exc + return prompts + + +def _changed_snapshot_files(snapshot: Any) -> list[str]: + changed: list[str] = [] + for name, prompt_file in snapshot.files.items(): + try: + current = prompt_file.path.read_bytes() + except OSError: + changed.append(name) + else: + if current != prompt_file.content: + changed.append(name) + return changed + + +def _round_id(round_record: Any, *, fallback: int) -> int: + value = getattr(round_record, "round", fallback) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError("SDK round id must be a positive integer") + return value + + +def _require_successful_optimize_result(result: Any) -> None: + status = _sdk_result_text(getattr(result, "status", None)).upper() + if status == "SUCCEEDED": + return + raise ValueError("SDK optimization did not succeed: " + f"status={status}; " + f"error_message={_sdk_result_text(getattr(result, 'error_message', None))}; " + f"finish_reason={_sdk_result_text(getattr(result, 'finish_reason', None))}; " + f"stop_reason={_sdk_result_text(getattr(result, 'stop_reason', None))}") + + +def _sdk_result_text(value: Any) -> str: + enum_value = getattr(value, "value", None) + if enum_value is not None: + value = enum_value + else: + enum_name = getattr(value, "name", None) + if enum_name is not None: + value = enum_name + return str(value).split(".")[-1] + + +def _prompt_bundle_key(prompts: dict[str, str]) -> tuple[tuple[str, str], ...]: + return tuple(sorted(prompts.items())) + + +def _candidate_from_bundle( + *, + candidate_id: str, + prompts: dict[str, str], + rationale: str, + baseline_prompts: dict[str, str], +) -> CandidatePrompt: + return CandidatePrompt( + candidate_id=candidate_id, + prompt=_render_prompt_bundle(prompts), + rationale=rationale, + prompt_diff=_render_prompt_bundle_diff( + baseline_prompts, + prompts, + candidate_id=candidate_id, + ), + prompt_fields=dict(prompts), + ) + + +def _summarize_sdk_result(result: Any) -> dict[str, Any]: + return { + "schema_version": + _safe_jsonable(getattr(result, "schema_version", None)), + "algorithm": + _safe_jsonable(getattr(result, "algorithm", None)), + "status": + _safe_jsonable(getattr(result, "status", None)), + "finish_reason": + _safe_jsonable(getattr(result, "finish_reason", None)), + "stop_reason": + _safe_jsonable(getattr(result, "stop_reason", None)), + "error_message": + _safe_jsonable(getattr(result, "error_message", None)), + "baseline_pass_rate": + _pass_rate_result_field( + "baseline_pass_rate", + getattr(result, "baseline_pass_rate", 0.0), + ), + "best_pass_rate": + _pass_rate_result_field( + "best_pass_rate", + getattr(result, "best_pass_rate", 0.0), + ), + "pass_rate_improvement": + _finite_result_field( + "pass_rate_improvement", + getattr(result, "pass_rate_improvement", 0.0), + ), + "baseline_metric_breakdown": + _finite_metric_map( + getattr(result, "baseline_metric_breakdown", {}) or {}, + context="SDK OptimizeResult baseline_metric_breakdown", + ), + "best_metric_breakdown": + _finite_metric_map( + getattr(result, "best_metric_breakdown", {}) or {}, + context="SDK OptimizeResult best_metric_breakdown", + ), + "metric_thresholds": + _finite_metric_map( + getattr(result, "metric_thresholds", {}) or {}, + context="SDK OptimizeResult metric_thresholds", + ), + "per_metric_best_candidates": + _safe_jsonable(getattr(result, "per_metric_best_candidates", {})), + "total_llm_cost": + _nonnegative_result_field( + "total_llm_cost", + getattr(result, "total_llm_cost", 0.0), + ), + "total_token_usage": + _safe_jsonable(getattr(result, "total_token_usage", {})), + "duration_seconds": + _nonnegative_result_field( + "duration_seconds", + getattr(result, "duration_seconds", 0.0), + ), + "started_at": + _safe_jsonable(getattr(result, "started_at", None)), + "finished_at": + _safe_jsonable(getattr(result, "finished_at", None)), + "total_rounds": + _safe_jsonable(getattr(result, "total_rounds", 0)), + "baseline_prompts": + _safe_jsonable(getattr(result, "baseline_prompts", {})), + "best_prompts": + _safe_jsonable(getattr(result, "best_prompts", {})), + "rounds": [_round_raw_summary(round_record) for round_record in getattr(result, "rounds", []) or []], + "extras": + _safe_jsonable(getattr(result, "extras", {})), + } + + +def _round_raw_summary(round_record: Any) -> dict[str, Any]: + serialized = _safe_jsonable(round_record) + if isinstance(serialized, dict): + return serialized + return { + "round": _safe_jsonable(getattr(round_record, "round", None)), + "candidate_prompts": _safe_jsonable(getattr(round_record, "candidate_prompts", {})), + "validation_pass_rate": _safe_jsonable(getattr(round_record, "validation_pass_rate", None)), + "metric_breakdown": _safe_jsonable(getattr(round_record, "metric_breakdown", {})), + "accepted": _safe_jsonable(getattr(round_record, "accepted", None)), + "acceptance_reason": _safe_jsonable(getattr(round_record, "acceptance_reason", "")), + "failed_case_ids": _safe_jsonable(getattr(round_record, "failed_case_ids", [])), + "round_llm_cost": _safe_jsonable(getattr(round_record, "round_llm_cost", 0.0)), + "duration_seconds": _safe_jsonable(getattr(round_record, "duration_seconds", 0.0)), + } + + +def _finite_result_field(field_name: str, value: Any) -> float: + try: + return _finite_number(value, context=f"SDK OptimizeResult field {field_name}") + except ValueError as exc: + raise ValueError(f"SDK OptimizeResult field {field_name} must be a finite number") from exc + + +def _nonnegative_result_field(field_name: str, value: Any) -> float: + number = _finite_result_field(field_name, value) + if number < 0.0: + raise ValueError(f"SDK OptimizeResult field {field_name} must be non-negative") + return number + + +def _pass_rate_result_field(field_name: str, value: Any) -> float: + number = _finite_result_field(field_name, value) + if not 0.0 <= number <= 1.0: + raise ValueError(f"SDK OptimizeResult field {field_name} must be between 0 and 1") + return number + + +def _finite_metric_map(value: Any, *, context: str) -> dict[str, float]: + if not isinstance(value, dict): + raise ValueError(f"{context} must be a metric mapping") + return {str(name): _finite_number(score, context=f"{context}.{name}") for name, score in value.items()} + + +def _finite_number(value: Any, *, context: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{context} must be a finite number") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{context} must be a finite number") + return number + + +def _positive_perf_duration(started_at: float, finished_at: float) -> float: + """Return one finite positive duration for a measured perf-counter span.""" + + elapsed = finished_at - started_at + if math.isfinite(elapsed) and elapsed > 0.0: + return elapsed + + resolution = float(time.get_clock_info("perf_counter").resolution) + if not math.isfinite(resolution) or resolution <= 0.0: + raise RuntimeError("perf_counter resolution must be finite and positive") + return resolution + + +def _nonnegative_number(value: Any, *, context: str) -> float: + number = _finite_number(value, context=context) + if number < 0.0: + raise ValueError(f"{context} must be non-negative") + return number + + +def _pass_rate_number(value: Any, *, context: str) -> float: + number = _finite_number(value, context=context) + if not 0.0 <= number <= 1.0: + raise ValueError(f"{context} must be between 0 and 1") + return number + + +def _safe_jsonable(value: Any) -> Any: + if hasattr(value, "model_dump"): + return _safe_jsonable(value.model_dump(mode="json")) + if hasattr(value, "__dict__") and not isinstance(value, type): + return _safe_jsonable(dict(value.__dict__)) + if isinstance(value, dict): + return {str(key): _safe_jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_safe_jsonable(item) for item in value] + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("SDK result values must be finite") + return value + if isinstance(value, (str, int, bool)) or value is None: + return value + return repr(value) + + +def _load_sdk_expected_cases( + dataset_path: str | Path, + *, + split: str, +) -> dict[str, EvalCase]: + """Load wrapper metadata from an SDK EvalSet without changing the SDK file.""" + + payload = read_json(dataset_path) + if "eval_cases" not in payload: + if "cases" in payload: + return _eval_cases_by_id( + load_eval_cases(dataset_path, split=split), + context=f"legacy evalset {dataset_path}", + ) + raise ValueError(f"SDK evalset {dataset_path} must contain an eval_cases list") + + eval_set_id = payload.get("eval_set_id") + if not isinstance(eval_set_id, str) or not eval_set_id.strip(): + raise ValueError(f"SDK evalset {dataset_path} is missing non-empty eval_set_id") + raw_cases = payload["eval_cases"] + if not isinstance(raw_cases, list): + raise ValueError(f"SDK evalset {dataset_path} eval_cases must be a list") + if not raw_cases: + raise ValueError(f"SDK evalset {dataset_path} eval_cases must not be empty") + + expected_cases: dict[str, EvalCase] = {} + for index, raw_case in enumerate(raw_cases): + if not isinstance(raw_case, dict): + raise ValueError(f"SDK evalset {dataset_path} eval_cases[{index}] must be an object") + eval_id = raw_case.get("eval_id") + if not isinstance(eval_id, str) or not eval_id.strip(): + raise ValueError(f"SDK evalset {dataset_path} eval_cases[{index}] is missing non-empty eval_id") + if eval_id in expected_cases: + raise ValueError(f"SDK evalset {dataset_path} contains duplicate eval_id {eval_id!r}") + expected_cases[eval_id] = _expected_case_from_sdk_case( + raw_case, + eval_id=eval_id, + split=split, + dataset_path=dataset_path, + ) + return expected_cases + + +def _expected_case_from_sdk_case( + raw_case: dict[str, Any], + *, + eval_id: str, + split: str, + dataset_path: str | Path, +) -> EvalCase: + context = f"SDK evalset {dataset_path} case {eval_id!r}" + conversation = raw_case.get("conversation") + if not isinstance(conversation, list) or not conversation: + raise ValueError(f"{context} must contain a non-empty conversation list") + + input_text = "" + for turn_index, invocation in reversed(list(enumerate(conversation))): + if not isinstance(invocation, dict): + raise ValueError(f"{context} conversation[{turn_index}] must be an object") + user_content = invocation.get("user_content") + if user_content is None: + continue + if not isinstance(user_content, dict): + raise ValueError(f"{context} conversation[{turn_index}].user_content must be an object") + candidate_text = _content_text(user_content) + if candidate_text.strip(): + input_text = candidate_text + break + if not input_text: + raise ValueError(f"{context} conversation has no user_content text") + + session_input = raw_case.get("session_input") + if not isinstance(session_input, dict): + raise ValueError(f"{context} must contain a session_input object") + state = session_input.get("state") + if not isinstance(state, dict): + raise ValueError(f"{context} session_input.state must be an object") + expectation = state.get("eval_optimize_expectation") + if not isinstance(expectation, dict): + raise ValueError(f"{context} session_input.state must contain eval_optimize_expectation object") + + tags = state.get("eval_optimize_tags", []) + if not isinstance(tags, list) or any(not isinstance(tag, str) for tag in tags): + raise ValueError(f"{context} eval_optimize_tags must be a list of strings") + protected = state.get("eval_optimize_protected", False) + if not isinstance(protected, bool): + raise ValueError(f"{context} eval_optimize_protected must be a boolean") + + expected_failure_category = state.get("eval_optimize_expected_failure_category") + if expected_failure_category is None: + expected_failure_category = state.get("expected_failure_category") + if expected_failure_category is None: + expected_failure_category = expectation.get("expected_failure_category") + if (expected_failure_category is not None + and (not isinstance(expected_failure_category, str) or not expected_failure_category.strip())): + raise ValueError(f"{context} expected_failure_category must be a non-empty string") + + return EvalCase( + case_id=eval_id, + split=split, + input=input_text, + expectation=dict(expectation), + tags=list(tags), + protected=protected, + expected_failure_category=expected_failure_category, + ) + + +def _eval_cases_by_id( + cases: Iterable[EvalCase], + *, + context: str, +) -> dict[str, EvalCase]: + by_id: dict[str, EvalCase] = {} + for case in cases: + if case.case_id in by_id: + raise ValueError(f"{context} contains duplicate case id {case.case_id!r}") + by_id[case.case_id] = case + return by_id + + +def _eval_result_from_sdk_result( + result: Any, + *, + prompt_id: str, + split: str, + expected_cases: Iterable[EvalCase], +) -> EvalResult: + expected_by_id: dict[str, EvalCase] = {} + expected_order: list[str] = [] + for expected_case in expected_cases: + case_id = str(expected_case.case_id) + if case_id in expected_by_id: + raise ValueError(f"expected cases contain duplicate case id: {case_id}") + expected_by_id[case_id] = expected_case + expected_order.append(case_id) + + sdk_runs_by_id: dict[str, list[Any]] = {} + results_by_eval_set_id = getattr(result, "results_by_eval_set_id", {}) or {} + if not results_by_eval_set_id: + raise ValueError("SDK EvaluateResult contains no eval set results") + for raw_eval_set_id, set_result in results_by_eval_set_id.items(): + eval_set_id = str(raw_eval_set_id) + num_runs = _optional_num_runs(set_result, eval_set_id=eval_set_id) + eval_results_by_eval_id = getattr(set_result, "eval_results_by_eval_id", {}) or {} + for raw_eval_id, runs in eval_results_by_eval_id.items(): + eval_id = str(raw_eval_id) + if eval_id in sdk_runs_by_id: + raise ValueError(f"SDK evaluation result contains duplicate case id: {eval_id}") + run_list = list(runs or []) + if num_runs is not None and len(run_list) != num_runs: + raise ValueError(f"SDK eval set {eval_set_id!r} declares num_runs={num_runs}, " + f"but case {eval_id!r} contains {len(run_list)} runs") + _validate_sdk_run_ids( + run_list, + eval_set_id=eval_set_id, + eval_id=eval_id, + num_runs=num_runs, + ) + sdk_runs_by_id[eval_id] = run_list + + expected_ids = set(expected_by_id) + sdk_ids = set(sdk_runs_by_id) + if expected_ids != sdk_ids: + details: list[str] = [] + missing = sorted(expected_ids - sdk_ids) + extra = sorted(sdk_ids - expected_ids) + if missing: + details.append("missing SDK result IDs: " + ", ".join(missing)) + if extra: + details.append("extra SDK result IDs: " + ", ".join(extra)) + raise ValueError("SDK evaluation case IDs do not match expected cases; " + "; ".join(details)) + + case_results: list[CaseResult] = [] + for case_id in expected_order: + expected_case = expected_by_id[case_id] + run_list = sdk_runs_by_id[case_id] + metrics = _aggregate_case_metrics(run_list, case_id=case_id) + if metrics: + score = _mean(list(metrics.values())) + else: + score = _mean([1.0 if _status_passed(getattr(run, "final_eval_status", None)) else 0.0 for run in run_list]) + score = round(score, 6) + passed = bool(run_list) and all(_status_passed(getattr(run, "final_eval_status", None)) for run in run_list) + failure_reason, evidence, failure_category = _failure_details(run_list) + actual_invocation = _last_actual_invocation(run_list) + trace_available = actual_invocation is not None + trace_payload = ({ + "user_content": _safe_jsonable(getattr(actual_invocation, "user_content", None)), + "final_response": _safe_jsonable(getattr(actual_invocation, "final_response", None)), + "intermediate_data": _safe_jsonable(getattr(actual_invocation, "intermediate_data", None)), + } if actual_invocation is not None else {}) + output = (_content_text(getattr(actual_invocation, "final_response", None)) + if actual_invocation is not None else "") + case_results.append( + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=passed, + output=output, + metrics=metrics, + trace=trace_payload, + trace_available=trace_available, + failure_category=None if passed else failure_category, + failure_reason=None if passed else failure_reason, + evidence=None if passed else evidence, + cost=0.0, + hard_failed=(not passed and score <= 0.0), + expected_failure_category=expected_case.expected_failure_category, + )) + + aggregate_score = (round(_mean([case.score for case in case_results]), 6) if case_results else 0.0) + return EvalResult( + prompt_id=prompt_id, + split=split, + score=aggregate_score, + passed=all(case.passed for case in case_results), + cost=0.0, + cases=case_results, + ) + + +def _optional_num_runs(set_result: Any, *, eval_set_id: str) -> int | None: + if not hasattr(set_result, "num_runs"): + return None + num_runs = getattr(set_result, "num_runs") + if isinstance(num_runs, bool) or not isinstance(num_runs, int) or num_runs <= 0: + raise ValueError(f"SDK eval set {eval_set_id!r} num_runs must be a positive integer") + return num_runs + + +def _validate_sdk_run_ids( + runs: list[Any], + *, + eval_set_id: str, + eval_id: str, + num_runs: int | None, +) -> None: + seen_run_ids: set[int] = set() + for run in runs: + internal_eval_id = getattr(run, "eval_id", None) + if internal_eval_id not in (None, "") and str(internal_eval_id) != eval_id: + raise ValueError(f"SDK run internal eval_id {internal_eval_id!r} does not match " + f"container case id {eval_id!r}") + internal_eval_set_id = getattr(run, "eval_set_id", None) + if (internal_eval_set_id not in (None, "") and str(internal_eval_set_id) != eval_set_id): + raise ValueError(f"SDK run internal eval_set_id {internal_eval_set_id!r} does not match " + f"container eval set id {eval_set_id!r}") + + run_id = getattr(run, "run_id", None) + if run_id is None: + continue + if isinstance(run_id, bool) or not isinstance(run_id, int) or run_id <= 0: + raise ValueError(f"SDK case {eval_id!r} run_id must be a positive integer or None") + if num_runs is not None and run_id > num_runs: + raise ValueError(f"SDK case {eval_id!r} run_id {run_id} exceeds num_runs={num_runs}") + if run_id in seen_run_ids: + raise ValueError(f"SDK evaluation result contains duplicate run_id {run_id} for case {eval_id}") + seen_run_ids.add(run_id) + + +def _aggregate_case_metrics(runs: list[Any], *, case_id: str) -> dict[str, float]: + scores_by_metric: dict[str, list[float]] = {} + for run in runs: + for metric in getattr(run, "overall_eval_metric_results", []) or []: + raw_score = getattr(metric, "score", None) + if raw_score is None: + continue + metric_name = str(getattr(metric, "metric_name", "") or "") + if not metric_name: + raise ValueError(f"SDK case {case_id} contains a scored metric without a name") + score = _finite_number( + raw_score, + context=f"SDK case {case_id} metric {metric_name} score", + ) + scores_by_metric.setdefault(metric_name, []).append(score) + return {metric_name: round(_mean(scores), 6) for metric_name, scores in scores_by_metric.items()} + + +def _mean(values: list[float]) -> float: + return sum(values) / len(values) if values else 0.0 + + +def _status_passed(status: Any) -> bool: + name = getattr(status, "name", None) + if name: + return str(name).upper() == "PASSED" + return str(status).split(".")[-1].upper() == "PASSED" + + +def _failure_details(runs: list[Any]) -> tuple[str, str, str]: + for run in runs: + if _status_passed(getattr(run, "final_eval_status", None)): + continue + error_message = getattr(run, "error_message", None) + for metric in getattr(run, "overall_eval_metric_results", []) or []: + if _status_passed(getattr(metric, "eval_status", None)): + continue + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details is not None else None + metric_name = str(getattr(metric, "metric_name", "") or "") + score = getattr(metric, "score", None) + evidence = f"{metric_name} score={score}" if metric_name else f"score={score}" + return ( + str(reason or error_message or "evaluation metric failed"), + evidence, + _metric_failure_category(metric_name), + ) + if error_message: + return (str(error_message), str(error_message), "evaluation_error") + return ("evaluation failed", "no failed metric detail available", "unknown_failure") + + +def _metric_failure_category(metric_name: str) -> str: + lowered = metric_name.lower() + if "parameter" in lowered or "arg" in lowered: + return "parameter_error" + if "tool" in lowered: + return "tool_call_error" + if "knowledge" in lowered or "recall" in lowered: + return "knowledge_recall_insufficient" + if "rubric" in lowered or "judge" in lowered or "llm" in lowered: + return "llm_rubric_not_met" + if "format" in lowered: + return "format_violation" + if "response" in lowered or "match" in lowered or "exact" in lowered: + return "final_response_mismatch" + return "unknown_failure" + + +def _last_actual_invocation(runs: list[Any]) -> Any | None: + for run in reversed(runs): + invocation_results = getattr(run, "eval_metric_result_per_invocation", []) or [] + for invocation_result in reversed(invocation_results): + actual = getattr(invocation_result, "actual_invocation", None) + if actual is not None: + return actual + return None + + +def _content_text(content: Any) -> str: + if content is None: + return "" + if hasattr(content, "model_dump"): + content = content.model_dump(mode="json") + if not isinstance(content, dict): + return "" + texts = [] + for part in content.get("parts") or []: + if isinstance(part, dict) and part.get("text") is not None: + texts.append(str(part["text"])) + return "\n".join(texts) + + +def _render_prompt_bundle(prompts: dict[str, str]) -> str: + if set(prompts) == {"system_prompt"}: + return prompts["system_prompt"] + sections = [] + for name in sorted(prompts): + sections.append(f"## {name}\n\n{prompts[name]}") + return "\n\n".join(sections) + + +def _render_prompt_bundle_diff( + baseline_prompts: dict[str, str], + candidate_prompts: dict[str, str], + *, + candidate_id: str, +) -> str: + if set(baseline_prompts) == {"system_prompt"} and set(candidate_prompts) == {"system_prompt"}: + return make_unified_diff( + baseline_prompts.get("system_prompt", ""), + candidate_prompts.get("system_prompt", ""), + before_name="baseline_system_prompt.txt", + after_name=f"{candidate_id}/system_prompt.txt", + ) + diffs = [] + for name in sorted(set(baseline_prompts) | set(candidate_prompts)): + diffs.append( + make_unified_diff( + baseline_prompts.get(name, ""), + candidate_prompts.get(name, ""), + before_name=f"baseline/{name}.txt", + after_name=f"{candidate_id}/{name}.txt", + )) + return "\n\n".join(diffs) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/config.py b/examples/optimization/eval_optimize_loop/eval_loop/config.py new file mode 100644 index 00000000..4af66972 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/config.py @@ -0,0 +1,243 @@ +"""Configuration validation for the eval/optimize loop example.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from dataclasses import field +from pathlib import Path +from typing import Any + +from .artifacts import validate_distinct_file_paths +from .schemas import EvalCase + + +@dataclass(frozen=True) +class GateConfig: + min_val_score_improvement: float = 0.01 + allow_new_hard_fail: bool = False + protected_case_ids: list[str] = field(default_factory=list) + max_score_drop_per_case: float = 0.0 + max_total_cost: float | None = 1.0 + extras: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + data = { + "min_val_score_improvement": self.min_val_score_improvement, + "allow_new_hard_fail": self.allow_new_hard_fail, + "protected_case_ids": list(self.protected_case_ids), + "max_score_drop_per_case": self.max_score_drop_per_case, + "max_total_cost": self.max_total_cost, + } + data.update(self.extras) + return data + + +@dataclass(frozen=True) +class OptimizerConfig: + seed: int = 91 + optimizer: dict[str, Any] = field(default_factory=dict) + metrics: dict[str, Any] = field(default_factory=dict) + gate: GateConfig = field(default_factory=GateConfig) + extras: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + data = { + "seed": self.seed, + "optimizer": dict(self.optimizer), + "metrics": dict(self.metrics), + "gate": self.gate.to_dict(), + } + data.update(self.extras) + return data + + +def parse_optimizer_config(payload: dict[str, Any], *, path: str | Path) -> OptimizerConfig: + path_text = str(path) + allowed = {"seed", "optimizer", "metrics", "gate"} + extras = {key: value for key, value in payload.items() if key not in allowed} + + seed = resolve_effective_seed(payload, path=path) + + optimizer = payload.get("optimizer", {}) + if not isinstance(optimizer, dict): + raise ValueError(f"{path_text}: field 'optimizer' must be an object") + + metrics = payload.get("metrics", {}) + if not isinstance(metrics, dict): + raise ValueError(f"{path_text}: field 'metrics' must be an object") + + gate_payload = payload.get("gate", {}) + if not isinstance(gate_payload, dict): + raise ValueError(f"{path_text}: field 'gate' must be an object") + + gate = _parse_gate_config(gate_payload, path=path_text) + return OptimizerConfig( + seed=seed, + optimizer=dict(optimizer), + metrics=dict(metrics), + gate=gate, + extras=extras, + ) + + +def resolve_effective_seed( + payload: dict[str, Any], + *, + path: str | Path, + default: int = 91, + strict_legacy: bool = True, +) -> int: + """Resolve the legacy or official optimizer seed without audit drift. + + The official SDK schema owns ``optimize.algorithm.seed``. A non-integer + top-level ``seed`` may be unrelated SDK metadata and is ignored when the + official nested seed is present. Two integer seed declarations must agree. + """ + + path_text = str(path) + nested_present = False + nested_seed: Any = None + optimize = payload.get("optimize") + if isinstance(optimize, dict): + algorithm = optimize.get("algorithm") + if isinstance(algorithm, dict) and "seed" in algorithm: + nested_present = True + nested_seed = algorithm["seed"] + + top_present = "seed" in payload + top_seed = payload.get("seed") + if nested_present: + nested = _validated_seed( + nested_seed, + field_name=f"{path_text}: field 'optimize.algorithm.seed'", + ) + if top_present and isinstance(top_seed, int) and not isinstance(top_seed, bool): + if top_seed != nested: + raise ValueError(f"{path_text}: conflicting seed values: top-level seed={top_seed}, " + f"optimize.algorithm.seed={nested}") + return nested + + if not top_present: + return default + if not strict_legacy and (isinstance(top_seed, bool) or not isinstance(top_seed, int)): + return default + return _validated_seed(top_seed, field_name=f"{path_text}: field 'seed'") + + +def _validated_seed(value: Any, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{field_name} must be an integer") + return value + + +def validate_inputs( + *, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + train_cases: list[EvalCase], + validation_cases: list[EvalCase], + config: OptimizerConfig, +) -> None: + validate_distinct_file_paths( + { + "train": train_path, + "validation": val_path + }, + context="train and validation evalset paths", + ) + + _validate_cases(train_cases, split="train", path=train_path) + _validate_cases(validation_cases, split="validation", path=val_path) + if len(train_cases) < 3: + raise ValueError(f"{train_path}: train evalset must contain at least 3 cases") + if len(validation_cases) < 3: + raise ValueError(f"{val_path}: validation evalset must contain at least 3 cases") + + validation_ids = {case.case_id for case in validation_cases} + missing_protected = [case_id for case_id in config.gate.protected_case_ids if case_id not in validation_ids] + if missing_protected: + raise ValueError( + f"{optimizer_config_path}: field 'gate.protected_case_ids' references missing validation cases: " + f"{missing_protected}") + + +def _parse_gate_config(payload: dict[str, Any], *, path: str) -> GateConfig: + allowed = { + "min_val_score_improvement", + "allow_new_hard_fail", + "protected_case_ids", + "max_score_drop_per_case", + "max_total_cost", + } + extras = {key: value for key, value in payload.items() if key not in allowed} + + min_val = _finite_number( + payload.get("min_val_score_improvement", 0.01), + f"{path}: field 'gate.min_val_score_improvement'", + 0.0, + 1.0, + ) + + allow_new_hard_fail = payload.get("allow_new_hard_fail", False) + if not isinstance(allow_new_hard_fail, bool): + raise ValueError(f"{path}: field 'gate.allow_new_hard_fail' must be a boolean") + + protected_case_ids = payload.get("protected_case_ids", []) + if not isinstance(protected_case_ids, list) or not all(isinstance(item, str) for item in protected_case_ids): + raise ValueError(f"{path}: field 'gate.protected_case_ids' must be a list of strings") + + max_drop = _finite_number( + payload.get("max_score_drop_per_case", 0.0), + f"{path}: field 'gate.max_score_drop_per_case'", + 0.0, + ) + + max_total_cost_value = payload.get("max_total_cost", 1.0) + max_total_cost = (None if max_total_cost_value is None else _finite_number( + max_total_cost_value, + f"{path}: field 'gate.max_total_cost'", + 0.0, + )) + + return GateConfig( + min_val_score_improvement=min_val, + allow_new_hard_fail=allow_new_hard_fail, + protected_case_ids=list(protected_case_ids), + max_score_drop_per_case=max_drop, + max_total_cost=max_total_cost, + extras=extras, + ) + + +def _finite_number( + value: Any, + field_name: str, + minimum: float, + maximum: float | None = None, +) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field_name} must be a finite number") + try: + number = float(value) + except OverflowError as exc: + raise ValueError(f"{field_name} must be a finite number") from exc + if not math.isfinite(number): + raise ValueError(f"{field_name} must be a finite number") + if number < minimum: + raise ValueError(f"{field_name} must be a finite number greater than or equal to {minimum:g}") + if maximum is not None and number > maximum: + raise ValueError(f"{field_name} must be a finite number between {minimum:g} and {maximum:g}") + return number + + +def _validate_cases(cases: list[EvalCase], *, split: str, path: str | Path) -> None: + seen: set[str] = set() + for case in cases: + if case.case_id in seen: + raise ValueError(f"{path}: duplicate case_id {case.case_id!r} in {split} evalset") + seen.add(case.case_id) + expectation_type = case.expectation.get("type") + if not isinstance(expectation_type, str): + raise ValueError(f"{path}: case {case.case_id!r} field 'expectation.type' must be a string") diff --git a/examples/optimization/eval_optimize_loop/eval_loop/diffing.py b/examples/optimization/eval_optimize_loop/eval_loop/diffing.py new file mode 100644 index 00000000..9d578450 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/diffing.py @@ -0,0 +1,19 @@ +"""Prompt diff helpers.""" + +from __future__ import annotations + +import difflib + + +def make_unified_diff(before: str, after: str, *, before_name: str, after_name: str) -> str: + """Return a stable unified diff for prompt text.""" + + diff = difflib.unified_diff( + before.splitlines(), + after.splitlines(), + fromfile=before_name, + tofile=after_name, + lineterm="", + ) + rendered = "\n".join(line.rstrip() for line in diff) + return rendered or f"--- {before_name}\n+++ {after_name}\n# no prompt changes" diff --git a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py new file mode 100644 index 00000000..3019a998 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py @@ -0,0 +1,76 @@ +"""Example-local evaluator adapter. + +This adapter intentionally keeps the surface small: it drives a model callable, +uses the fake judge for deterministic scoring, and returns dataclass results. +It can later be replaced with SDK AgentEvaluator integration. +""" + +from __future__ import annotations + +from typing import Iterable + +from .attribution import attribute_failure +from .fake_judge import FakeJudge +from .fake_model import FakeModel +from .schemas import CaseResult +from .schemas import EvalCase +from .schemas import EvalResult +from .trace import make_trace + + +class ExampleEvaluator: + """Evaluate prompt text against cases with fake model/judge components.""" + + def __init__(self, model: FakeModel, judge: FakeJudge, *, trace_enabled: bool = False) -> None: + self.model = model + self.judge = judge + self.trace_enabled = trace_enabled + + def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: + case_results: list[CaseResult] = [] + for case in cases: + output, model_trace, cost = self.model.generate(prompt_id, prompt, case.input) + judged = self.judge.score(case, output) + failure_category = None + failure_reason = None + evidence = None + if not judged.passed: + failure_category, failure_reason, evidence = attribute_failure( + judged.error_code or "unknown_failure", + judged.evidence or "", + ) + trace = make_trace( + self.trace_enabled, + prompt_id=prompt_id, + case_id=case.case_id, + model_trace=model_trace, + judge_trace=judged.trace or {}, + ) + case_results.append( + CaseResult( + case_id=case.case_id, + split=case.split, + score=round(judged.score, 6), + passed=judged.passed, + output=output, + metrics={"fake_judge_score": round(judged.score, 6)}, + trace=trace, + trace_available=bool(trace), + failure_category=failure_category, + failure_reason=failure_reason, + evidence=evidence, + cost=cost, + hard_failed=(not judged.passed and judged.score <= 0.0), + expected_failure_category=case.expected_failure_category, + )) + + score = round(sum(case.score for case in case_results) / len(case_results), 6) if case_results else 0.0 + total_cost = round(sum(case.cost for case in case_results), 6) + return EvalResult( + prompt_id=prompt_id, + split=split, + score=score, + passed=all(case.passed for case in case_results), + cost=total_cost, + cases=case_results, + ) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py new file mode 100644 index 00000000..73386fa2 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py @@ -0,0 +1,226 @@ +"""Deterministic fake judge for offline scoring.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from .schemas import EvalCase + + +@dataclass(frozen=True) +class JudgeOutcome: + score: float + passed: bool + error_code: str | None = None + evidence: str | None = None + trace: dict[str, Any] | None = None + + +class FakeJudge: + """Scores JSON, exact-answer, rubric, tool, and knowledge cases offline.""" + + def score(self, case: EvalCase, output: str) -> JudgeOutcome: + expectation_type = case.expectation.get("type") + if expectation_type == "json": + return self._score_json(case, output) + if expectation_type == "exact": + return self._score_exact(case, output) + if expectation_type == "rubric": + return self._score_rubric(case, output) + if expectation_type == "tool": + return self._score_tool(case, output) + if expectation_type == "knowledge": + return self._score_knowledge(case, output) + return JudgeOutcome( + score=0.0, + passed=False, + error_code="unknown_expectation", + evidence=f"unsupported expectation type: {expectation_type!r}", + trace={"expectation_type": expectation_type}, + ) + + def _score_json(self, case: EvalCase, output: str) -> JudgeOutcome: + try: + parsed = json.loads(output) + except json.JSONDecodeError as exc: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_parse_failure", + evidence=f"json parser failed at char {exc.pos}: {exc.msg}", + trace={ + "expectation_type": "json", + "valid_json": False + }, + ) + if not isinstance(parsed, dict): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_value_mismatch", + evidence=f"expected JSON object, got {type(parsed).__name__}", + trace={ + "expectation_type": "json", + "valid_json": True, + "object": False + }, + ) + required_keys = list(case.expectation.get("required_keys") or []) + for key in required_keys: + if key not in parsed: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="required_key_missing", + evidence=f"missing key {key!r}; got keys {sorted(parsed.keys())!r}", + trace={ + "expectation_type": "json", + "valid_json": True, + "missing_key": key + }, + ) + expected_values = dict(case.expectation.get("expected_values") or {}) + for key, expected_value in expected_values.items(): + actual_value = parsed.get(key) + if actual_value != expected_value: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_value_mismatch", + evidence=f"{key!r}: expected {expected_value!r}, got {actual_value!r}", + trace={ + "expectation_type": "json", + "valid_json": True, + "mismatch_key": key + }, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "json", "valid_json": True}) + + def _score_exact(self, case: EvalCase, output: str) -> JudgeOutcome: + expected = str(case.expectation.get("expected", "")) + if _normalize_exact(output) != _normalize_exact(expected): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="exact_answer_mismatch", + evidence=f"expected normalized {expected!r}, got {output!r}", + trace={ + "expectation_type": "exact", + "expected": expected + }, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "exact", "expected": expected}) + + def _score_rubric(self, case: EvalCase, output: str) -> JudgeOutcome: + lowered = output.lower() + forbidden = [str(item) for item in case.expectation.get("forbidden") or []] + for pattern in forbidden: + if pattern.lower() in lowered: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="forbidden_pattern", + evidence=f"forbidden pattern {pattern!r} was present", + trace={ + "expectation_type": "rubric", + "forbidden_pattern": pattern + }, + ) + + must_include = [str(item) for item in case.expectation.get("must_include") or []] + missing = [term for term in must_include if term.lower() not in lowered] + if missing: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="missing_rubric_terms", + evidence=f"missing terms: {missing!r}", + trace={ + "expectation_type": "rubric", + "missing_terms": missing + }, + ) + + max_chars = case.expectation.get("max_chars") + if max_chars is not None and len(output) > int(max_chars): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="max_chars_exceeded", + evidence=f"length {len(output)} exceeded max_chars {max_chars}", + trace={ + "expectation_type": "rubric", + "length": len(output), + "max_chars": int(max_chars) + }, + ) + + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "rubric"}) + + def _score_tool(self, case: EvalCase, output: str) -> JudgeOutcome: + try: + parsed = json.loads(output) + except json.JSONDecodeError as exc: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="tool_call_error", + evidence=f"tool output was not JSON at char {exc.pos}: {exc.msg}", + trace={ + "expectation_type": "tool", + "valid_json": False + }, + ) + expected_tool = case.expectation.get("expected_tool") + if parsed.get("tool") != expected_tool: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="tool_call_error", + evidence=f"expected tool {expected_tool!r}, got {parsed.get('tool')!r}", + trace={ + "expectation_type": "tool", + "expected_tool": expected_tool + }, + ) + expected_args = dict(case.expectation.get("expected_args") or {}) + actual_args = parsed.get("args") or {} + for key, expected_value in expected_args.items(): + if actual_args.get(key) != expected_value: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="parameter_error", + evidence=f"arg {key!r}: expected {expected_value!r}, got {actual_args.get(key)!r}", + trace={ + "expectation_type": "tool", + "arg": key + }, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "tool"}) + + def _score_knowledge(self, case: EvalCase, output: str) -> JudgeOutcome: + lowered = output.lower() + required_sources = [str(item) for item in case.expectation.get("required_sources") or []] + required_terms = [str(item) for item in case.expectation.get("must_include_knowledge_terms") or []] + missing_sources = [source for source in required_sources if source.lower() not in lowered] + missing_terms = [term for term in required_terms if term.lower() not in lowered] + if missing_sources or missing_terms: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="knowledge_recall_insufficient", + evidence=f"missing sources={missing_sources!r}, terms={missing_terms!r}", + trace={ + "expectation_type": "knowledge", + "missing_sources": missing_sources, + "missing_terms": missing_terms, + }, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "knowledge"}) + + +def _normalize_exact(value: str) -> str: + return " ".join(value.strip().lower().split()) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py new file mode 100644 index 00000000..0588648d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py @@ -0,0 +1,106 @@ +"""Deterministic fake model used by the example pipeline.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Any + +_ASSIGNMENT_PATTERN = re.compile(r"(? None: + self.seed = seed + + def generate( + self, + prompt_id: str, + prompt: str, + user_input: str, + ) -> tuple[str, dict[str, Any], float]: + if not isinstance(user_input, str): + raise TypeError("user_input must be a string") + + mode = self._mode(prompt) + request = self._parse_request(user_input) + output = self._render(request, mode=mode) + trace = { + "seed": self.seed, + "prompt_id": prompt_id, + "prompt_mode": mode, + } + return output, trace, self.COST_PER_CALL + + @staticmethod + def _mode(prompt: str) -> str: + if _OVERFIT_INSTRUCTION in prompt: + return "overfit" + if _SAFE_INSTRUCTION in prompt: + return "safe" + return "baseline" + + @staticmethod + def _parse_request(user_input: str) -> _ParsedRequest: + assignments = {match.group(1): match.group(2) for match in _ASSIGNMENT_PATTERN.finditer(user_input)} + only_match = _RETURN_ONLY_PATTERN.search(user_input) + return _ParsedRequest( + assignments=assignments, + only_value=only_match.group(1) if only_match else None, + strict_json=bool(_STRICT_JSON_PATTERN.search(user_input)), + natural_answer=FakeModel._natural_answer(user_input), + ) + + @staticmethod + def _natural_answer(user_input: str) -> str: + lowered = user_input.lower() + if "latency" in lowered and "retries" in lowered: + return "Latency can trigger retries." + if "cache" in lowered and "stale data" in lowered: + return "Cache invalidation refreshes stale data." + return "Here is a natural response." + + @staticmethod + def _render(request: _ParsedRequest, *, mode: str) -> str: + if mode == "overfit": + payload: dict[str, str] + if request.assignments: + payload = request.assignments + else: + payload = { + "answer": request.only_value or request.natural_answer, + } + return json.dumps(payload, sort_keys=True) + + if request.strict_json: + payload_json = json.dumps(request.assignments, sort_keys=True) + if mode == "safe": + return payload_json + return f"Here is the JSON you requested: {payload_json}" + + if request.only_value is not None: + return request.only_value + + return request.natural_answer diff --git a/examples/optimization/eval_optimize_loop/eval_loop/gate.py b/examples/optimization/eval_optimize_loop/eval_loop/gate.py new file mode 100644 index 00000000..fd199d5c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/gate.py @@ -0,0 +1,189 @@ +"""Configurable acceptance gate for candidate prompts.""" + +from __future__ import annotations + +from typing import Any + +from .schemas import CaseDelta +from .schemas import CostSummary +from .schemas import EvalResult +from .schemas import GateDecision + +DEFAULT_GATE_CONFIG = { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, +} + + +class AcceptanceGate: + """Apply deterministic safety and quality constraints to candidates.""" + + def __init__(self, config: dict[str, Any]) -> None: + merged = dict(DEFAULT_GATE_CONFIG) + merged.update(config or {}) + self.config = merged + + def decide( + self, + *, + candidate_id: str, + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_train: EvalResult, + candidate_validation: EvalResult, + deltas: list[CaseDelta], + cost_summary: CostSummary, + cumulative_cost: float = 0.0, + ) -> GateDecision: + train_delta = round(candidate_train.score - baseline_train.score, 6) + val_delta = round(candidate_validation.score - baseline_validation.score, 6) + candidate_cost = round(candidate_train.cost + candidate_validation.cost, 6) + reasons: list[str] = [] + + train_comparable = _append_comparability_reasons( + reasons, + split="train", + baseline=baseline_train, + candidate=candidate_train, + ) + validation_comparable = _append_comparability_reasons( + reasons, + split="validation", + baseline=baseline_validation, + candidate=candidate_validation, + ) + + overfit_detected = train_delta > 0 and val_delta <= 0 + if overfit_detected: + reasons.append("reject: overfit detected because train score improved but " + "validation score regressed or did not improve " + f"({train_delta:+.3f} train, {val_delta:+.3f} validation)") + + min_val_improvement = float(self.config["min_val_score_improvement"]) + if val_delta < min_val_improvement: + reasons.append("reject: validation improvement " + f"{val_delta:+.3f} is below required {min_val_improvement:+.3f}") + + validation_new_failures: list[str] = [] + if validation_comparable: + baseline_validation_by_id = baseline_validation.by_case_id() + candidate_validation_by_id = candidate_validation.by_case_id() + validation_new_failures = [ + case_id for case_id, candidate_case in sorted(candidate_validation_by_id.items()) + if not candidate_case.passed and baseline_validation_by_id[case_id].passed + ] + if validation_new_failures: + reasons.append(f"reject: new validation failures appeared: {validation_new_failures}") + + new_hard_failures: list[str] = [] + for baseline_result, candidate_result, comparable in ( + (baseline_train, candidate_train, train_comparable), + (baseline_validation, candidate_validation, validation_comparable), + ): + if not comparable: + continue + baseline_by_id = baseline_result.by_case_id() + new_hard_failures.extend(case_id + for case_id, candidate_case in sorted(candidate_result.by_case_id().items()) + if candidate_case.hard_failed and not baseline_by_id[case_id].hard_failed) + new_hard_failures = sorted(set(new_hard_failures)) + if new_hard_failures and not bool(self.config["allow_new_hard_fail"]): + reasons.append(f"reject: new hard failures appeared: {new_hard_failures}") + + protected_ids = set(str(item) for item in self.config["protected_case_ids"]) + protected_regressions = [ + delta.case_id for delta in deltas + if delta.split == "validation" and delta.case_id in protected_ids and delta.delta < 0 + ] + if protected_regressions: + reasons.append(f"reject: protected cases regressed: {protected_regressions}") + + max_drop = float(self.config["max_score_drop_per_case"]) + excessive_drops = [delta.case_id for delta in deltas if delta.split == "validation" and delta.delta < -max_drop] + if excessive_drops: + reasons.append(f"reject: per-case validation score drops exceed {max_drop:.3f}: {excessive_drops}") + + total_run_cost = round(cumulative_cost + candidate_cost, 6) + configured_max_total_cost = self.config["max_total_cost"] + if configured_max_total_cost is not None: + max_total_cost = float(configured_max_total_cost) + if not cost_summary.complete: + reasons.append("reject: cost_unavailable for configured max_total_cost") + elif total_run_cost > max_total_cost: + reasons.append(f"reject: total run cost {total_run_cost:.3f} exceeds budget {max_total_cost:.3f}") + + accepted = not any(reason.startswith("reject:") for reason in reasons) + if accepted: + reasons.append("accept: validation score improved " + f"{val_delta:+.3f} with no protected regression or new hard failure") + + return GateDecision( + candidate_id=candidate_id, + accepted=accepted, + reasons=reasons, + train_score_delta=train_delta, + validation_score_delta=val_delta, + new_hard_failures=new_hard_failures, + protected_regressions=protected_regressions, + validation_new_failures=validation_new_failures, + excessive_score_drops=excessive_drops, + overfit_detected=overfit_detected, + candidate_cost=candidate_cost, + cumulative_cost=round(cumulative_cost, 6), + total_run_cost=total_run_cost, + cost=candidate_cost, + gate_status="applied", + gate_not_applied_reason=None, + not_applied_checks=[], + ) + + +def _append_comparability_reasons( + reasons: list[str], + *, + split: str, + baseline: EvalResult, + candidate: EvalResult, +) -> bool: + """Reject malformed result pairs before any dict conversion can hide evidence.""" + + comparable = True + baseline_ids = [case.case_id for case in baseline.cases] + candidate_ids = [case.case_id for case in candidate.cases] + if not baseline_ids: + reasons.append(f"reject: baseline {split} has empty case results") + comparable = False + if not candidate_ids: + reasons.append(f"reject: candidate {split} has empty case results") + comparable = False + + baseline_duplicates = _duplicates(baseline_ids) + candidate_duplicates = _duplicates(candidate_ids) + if baseline_duplicates: + reasons.append(f"reject: baseline {split} has duplicate case IDs: {baseline_duplicates}") + comparable = False + if candidate_duplicates: + reasons.append(f"reject: candidate {split} has duplicate case IDs: {candidate_duplicates}") + comparable = False + + baseline_set = set(baseline_ids) + candidate_set = set(candidate_ids) + if baseline_set != candidate_set: + reasons.append(f"reject: {split} case ID set mismatch; " + f"missing={sorted(baseline_set - candidate_set)}, " + f"extra={sorted(candidate_set - baseline_set)}") + comparable = False + return comparable + + +def _duplicates(case_ids: list[str]) -> list[str]: + seen: set[str] = set() + duplicates: set[str] = set() + for case_id in case_ids: + if case_id in seen: + duplicates.add(case_id) + seen.add(case_id) + return sorted(duplicates) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/loader.py b/examples/optimization/eval_optimize_loop/eval_loop/loader.py new file mode 100644 index 00000000..62659a16 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/loader.py @@ -0,0 +1,62 @@ +"""Input loading helpers for the deterministic optimization example.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .config import OptimizerConfig +from .config import parse_optimizer_config +from .schemas import EvalCase + + +def read_json(path: str | Path) -> dict[str, Any]: + resolved = Path(path) + try: + with resolved.open("r", encoding="utf-8") as file: + payload = json.load(file, parse_constant=_reject_non_standard_json_constant) + except (json.JSONDecodeError, ValueError) as exc: + raise ValueError(f"{resolved}: invalid JSON: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object in {resolved}") + return payload + + +def _reject_non_standard_json_constant(constant: str) -> None: + raise ValueError(f"non-standard JSON constant {constant!r}") + + +def load_eval_cases(path: str | Path, split: str | None = None) -> list[EvalCase]: + payload = read_json(path) + cases = payload.get("cases") + if not isinstance(cases, list): + raise ValueError(f"evalset {path} must contain a cases list") + effective_split = split or payload.get("split") or Path(path).name.split(".", 1)[0] + return [EvalCase.from_dict(case, str(effective_split)) for case in cases] + + +def load_optimizer_config(path: str | Path) -> OptimizerConfig: + payload = read_json(path) + return parse_optimizer_config(payload, path=path) + + +def load_prompt(path: str | Path) -> str: + return Path(path).read_text(encoding="utf-8") + + +def stable_config_hash(config: dict[str, Any]) -> str: + import hashlib + + canonical = json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def sha256_file(path: str | Path) -> str: + import hashlib + + digest = hashlib.sha256() + with Path(path).open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py new file mode 100644 index 00000000..0ac1559b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py @@ -0,0 +1,106 @@ +"""Fake optimizer that proposes deterministic prompt candidates.""" + +from __future__ import annotations + +from collections import Counter + +from .diffing import make_unified_diff +from .schemas import CandidatePrompt +from .schemas import EvalResult + +_TARGET_FAILURE_CATEGORIES = { + "format_violation", + "final_response_mismatch", +} +_OVERFIT_INSTRUCTION = "Always force every final answer into JSON." +_SAFE_INSTRUCTION = "Use strict JSON only when the user explicitly asks." + + +class FakeOptimizer: + """Propose candidates only for observed train formatting failures.""" + + def propose( + self, + baseline_prompt: str, + baseline_train: EvalResult, + failure_summary: dict[str, object], + ) -> list[CandidatePrompt]: + if not isinstance(failure_summary, dict): + raise TypeError("failure_summary must be a dict") + + if baseline_train.split != "train": + raise ValueError("baseline_train.split must be 'train'; " + f"got {baseline_train.split!r}") + for case in baseline_train.cases: + if case.split != "train": + raise ValueError(f"baseline_train case {case.case_id!r} split must be 'train'; " + f"got {case.split!r}") + + failed_cases = [case for case in baseline_train.cases if not case.passed] + observed_counts = Counter(case.failure_category for case in failed_cases if case.failure_category) + _validate_failure_summary(failure_summary, observed_counts) + + targeted = sorted(observed_counts.keys() & _TARGET_FAILURE_CATEGORIES) + if not targeted: + return [] + + overfit_prompt = f"{baseline_prompt.rstrip()}\n\n{_OVERFIT_INSTRUCTION}\n" + safe_prompt = f"{baseline_prompt.rstrip()}\n\n{_SAFE_INSTRUCTION}\n" + evidence = ", ".join(targeted) + return [ + CandidatePrompt( + candidate_id="candidate_001_overfit", + prompt=overfit_prompt, + rationale=(f"Observed training failures ({evidence}); this candidate deliberately " + "tests the risky global-JSON correction."), + prompt_diff=make_unified_diff( + baseline_prompt, + overfit_prompt, + before_name="baseline_system_prompt.txt", + after_name="candidate_001_overfit/system_prompt.txt", + ), + prompt_fields={"system_prompt": overfit_prompt}, + ), + CandidatePrompt( + candidate_id="candidate_002_safe", + prompt=safe_prompt, + rationale=(f"Observed training failures ({evidence}); this candidate limits strict " + "JSON behavior to explicit user requests."), + prompt_diff=make_unified_diff( + baseline_prompt, + safe_prompt, + before_name="baseline_system_prompt.txt", + after_name="candidate_002_safe/system_prompt.txt", + ), + prompt_fields={"system_prompt": safe_prompt}, + ), + ] + + +def _validate_failure_summary( + failure_summary: dict[str, object], + observed_counts: Counter[str], +) -> None: + if "by_category" not in failure_summary: + return + + by_category = failure_summary["by_category"] + if not isinstance(by_category, dict): + raise ValueError("failure_summary['by_category'] must be a dict") + + summary_counts: dict[object, int] = {} + for category, count in by_category.items(): + summary_counts[category] = _normalize_positive_count(category, count) + + if summary_counts != dict(observed_counts): + raise ValueError("failure_summary['by_category'] must match failed train cases exactly; " + f"summary={summary_counts!r}, observed={dict(observed_counts)!r}") + + +def _normalize_positive_count(category: object, value: object) -> int: + normalized = value if type(value) is int and value > 0 else None + + if normalized is None: + raise ValueError("failure_summary['by_category'] count must be a positive integer; " + f"category={category!r}, count={value!r}") + return normalized diff --git a/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py new file mode 100644 index 00000000..f9878ff4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/pipeline.py @@ -0,0 +1,1713 @@ +"""Backend-neutral asynchronous evaluation and optimization orchestration.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import tempfile +import time +from collections.abc import Awaitable +from collections.abc import Callable +from collections.abc import Mapping +from dataclasses import dataclass +from dataclasses import replace +from pathlib import Path +from types import MappingProxyType +from typing import Any + +from .attribution import summarize_failures +from .artifacts import validate_artifact_component +from .artifacts import validate_distinct_file_paths +from .backends import EvaluationBackend +from .backends import OptimizationBackend +from .config import _parse_gate_config +from .config import parse_optimizer_config +from .config import resolve_effective_seed +from .gate import AcceptanceGate +from .loader import read_json +from .loader import stable_config_hash +from .report import RunArtifactPaths +from .report import build_report +from .report import compute_case_deltas +from .report import finalize_run_artifacts +from .report import persist_writeback_journal +from .report import persist_writeback_outcome +from .report import prepare_run_artifacts +from .report import reserve_run_artifacts +from .report import validate_unique_round_ids +from .schemas import CandidatePrompt +from .schemas import CostSummary +from .schemas import EvalResult +from .schemas import GateDecision +from .schemas import OptimizationReport +from .schemas import OptimizationResult +from .schemas import WritebackResult +from .schemas import to_jsonable +from .writeback import commit_prompt_bundle +from .writeback import ConcurrentPromptUpdateError +from .writeback import PromptRestorationError +from .writeback import snapshot_prompt_files + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[4] +_SENSITIVE_CONFIG_KEY_SUFFIXES = ( + "apikey", + "authorization", + "authtoken", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "idtoken", + "password", + "passwd", + "privatekey", + "refreshtoken", + "secret", + "secretaccesskey", + "secretkey", + "signingkey", + "sshkey", + "token", + "accesstoken", +) +_SENSITIVE_CONFIG_KEY_MARKERS = ( + "apikey", + "password", + "passwd", + "credential", + "secret", + "authorization", + "privatekey", +) + + +@dataclass(frozen=True) +class PipelineRequest: + """All inputs required by the shared asynchronous pipeline.""" + + train_path: str | Path + validation_path: str | Path + optimizer_config_path: str | Path + output_dir: str | Path + target_prompt_paths: dict[str, str | Path] + gate_config: dict[str, Any] + trace: bool + update_source: bool + mode: str + run_id: str + sdk_call_agent: str | None = None + gate_config_path: str | Path | None = None + effective_seed: int | None = None + gate_config_source: str = "request" + + +@dataclass(frozen=True) +class _InputFileSnapshot: + role: str + original_path: Path + snapshot_path: Path + sha256: str + + +@dataclass(frozen=True) +class _InputSnapshots: + files: dict[str, _InputFileSnapshot] + + def path(self, role: str) -> Path: + return self.files[role].snapshot_path + + def hashes(self) -> dict[str, str]: + return {role: item.sha256 for role, item in self.files.items()} + + def cleanup(self) -> None: + failures: list[str] = [] + for item in self.files.values(): + try: + item.snapshot_path.unlink() + except FileNotFoundError: + pass + except OSError as error: + failures.append(f"{item.role}: {error}") + if failures: + raise OSError("failed to remove input snapshots: " + "; ".join(failures)) + + +async def execute_pipeline( + request: PipelineRequest, + *, + evaluator: EvaluationBackend, + optimizer: OptimizationBackend, +) -> OptimizationReport: + """Evaluate, optimize, gate, audit, and optionally commit one prompt bundle.""" + + started = time.perf_counter() + run_id = _safe_artifact_component(request.run_id, context="run_id") + if not request.target_prompt_paths: + raise ValueError("target_prompt_paths must not be empty") + _validate_target_prompt_fields(request.target_prompt_paths) + validate_distinct_file_paths( + { + "train": request.train_path, + "validation": request.validation_path + }, + context="train and validation evalset paths", + ) + artifact_paths = reserve_run_artifacts(request.output_dir, run_id=run_id) + run_root = artifact_paths.temp_run_dir + + input_snapshots = _snapshot_pipeline_inputs(request, run_id=run_id) + try: + result = await _execute_with_snapshots( + request, + evaluator=evaluator, + optimizer=optimizer, + started=started, + run_id=run_id, + run_root=run_root, + artifact_paths=artifact_paths, + input_snapshots=input_snapshots, + ) + except BaseException as primary_error: + try: + input_snapshots.cleanup() + except OSError as cleanup_error: + add_note = getattr(primary_error, "add_note", None) + if add_note is not None: + add_note(str(cleanup_error)) + raise + else: + input_snapshots.cleanup() + return result + + +def _validate_target_prompt_fields(target_prompt_paths: dict[str, str | Path]) -> None: + seen: set[str] = set() + for field_name, prompt_path in target_prompt_paths.items(): + try: + validate_artifact_component(field_name, context="target prompt field") + except ValueError as error: + raise ValueError(f"target prompt field {field_name!r} is unsafe") from error + + normalized_name = field_name.casefold() + if normalized_name in seen: + raise ValueError("target prompt field names must be case-insensitively unique") + seen.add(normalized_name) + + validate_distinct_file_paths( + target_prompt_paths, + context="target prompt fields", + ) + + +async def _execute_with_snapshots( + request: PipelineRequest, + *, + evaluator: EvaluationBackend, + optimizer: OptimizationBackend, + started: float, + run_id: str, + run_root: Path, + artifact_paths: RunArtifactPaths, + input_snapshots: _InputSnapshots, +) -> OptimizationReport: + prompt_snapshot = snapshot_prompt_files(request.target_prompt_paths) + baseline_prompts = MappingProxyType(_decode_snapshot(prompt_snapshot)) + raw_config = read_json(input_snapshots.path("optimizer")) + effective_seed = resolve_effective_seed( + raw_config, + path=request.optimizer_config_path, + strict_legacy=request.mode == "fake", + ) + if request.effective_seed is not None and request.effective_seed != effective_seed: + raise ValueError("effective optimizer seed changed between dependency construction and pipeline entry") + if request.mode == "fake": + _validate_backend_seed(evaluator, effective_seed) + if optimizer is not evaluator: + _validate_backend_seed(optimizer, effective_seed) + train_metadata = _strict_case_metadata(input_snapshots.path("train"), role="train") + validation_metadata = _strict_case_metadata( + input_snapshots.path("validation"), + role="validation", + ) + gate_config = _effective_gate_config(request, input_snapshots, raw_config=raw_config) + effective_gate_config = _gate_config_with_dataset_protection( + gate_config, + validation_metadata=validation_metadata, + ) + train_path = input_snapshots.path("train") + validation_path = input_snapshots.path("validation") + optimizer_path = input_snapshots.path("optimizer") + _verify_snapshot_integrity(input_snapshots) + + _verify_snapshot_integrity(input_snapshots, "train") + baseline_train = await _await_backend_with_prompt_integrity( + lambda: evaluator.evaluate( + prompt_id="baseline", + prompts=dict(baseline_prompts), + dataset_path=train_path, + split="train", + trace=request.trace, + artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "train"), + ), + prompt_snapshot=prompt_snapshot, + context="baseline train evaluation", + ) + _verify_snapshot_integrity(input_snapshots, "train") + _validate_eval_result( + baseline_train, + expected_prompt_id="baseline", + expected_split="train", + expected_case_ids=set(train_metadata), + ) + _verify_snapshot_integrity(input_snapshots, "validation") + baseline_validation = await _await_backend_with_prompt_integrity( + lambda: evaluator.evaluate( + prompt_id="baseline", + prompts=dict(baseline_prompts), + dataset_path=validation_path, + split="validation", + trace=request.trace, + artifact_dir=_artifact_dir(run_root, "evaluations", "000_baseline", "validation"), + ), + prompt_snapshot=prompt_snapshot, + context="baseline validation evaluation", + ) + _verify_snapshot_integrity(input_snapshots, "validation") + _validate_eval_result( + baseline_validation, + expected_prompt_id="baseline", + expected_split="validation", + expected_case_ids=set(validation_metadata), + ) + + failure_summary = summarize_failures([baseline_train]) + _verify_snapshot_integrity(input_snapshots, "train", "validation", "optimizer") + optimization = await _await_backend_with_prompt_integrity( + lambda: optimizer.optimize_candidates( + baseline_prompts=dict(baseline_prompts), + baseline_train=baseline_train, + failure_summary=failure_summary, + train_path=train_path, + validation_path=validation_path, + config_path=optimizer_path, + artifact_dir=_artifact_dir(run_root, "optimizer"), + ), + prompt_snapshot=prompt_snapshot, + context="candidate optimization", + ) + _verify_snapshot_integrity(input_snapshots, "train", "validation", "optimizer") + _validate_optimization_result(optimization) + candidate_bundles = _validated_candidate_bundles( + optimization.candidates, + expected_fields=set(prompt_snapshot.files), + ) + + gate = AcceptanceGate(effective_gate_config) + baseline_evaluator_cost = round(baseline_train.cost + baseline_validation.cost, 6) + explicit_evaluator_cost = baseline_evaluator_cost + candidate_records: list[dict[str, Any]] = [] + all_deltas = [] + gate_decisions = [] + all_results_comparable = True + candidate_evaluation_failed = False + + for index, candidate in enumerate(optimization.candidates, start=1): + bundle = candidate_bundles[candidate.candidate_id] + canonical_candidate = CandidatePrompt( + candidate_id=candidate.candidate_id, + prompt=candidate.prompt, + rationale=candidate.rationale, + prompt_diff=candidate.prompt_diff, + prompt_fields=dict(bundle), + ) + path_label = _candidate_artifact_component(index, candidate.candidate_id) + record: dict[str, Any] = { + "candidate": canonical_candidate, + "prompt_bundle": dict(bundle), + } + _verify_snapshot_integrity(input_snapshots, "train") + train_artifact_dir = _artifact_dir( + run_root, + "evaluations", + path_label, + "train", + ) + try: + train_result = await _await_backend_with_prompt_integrity( + lambda: evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=dict(bundle), + dataset_path=train_path, + split="train", + trace=request.trace, + artifact_dir=train_artifact_dir, + ), + prompt_snapshot=prompt_snapshot, + context=f"candidate {candidate.candidate_id!r} train evaluation", + ) + except Exception as error: + record["evaluation_error"] = _recoverable_candidate_evaluation_failure( + error, + stage="train", + completed_results=[], + input_snapshots=input_snapshots, + prompt_snapshot=prompt_snapshot, + ) + candidate_records.append(record) + candidate_evaluation_failed = True + all_results_comparable = False + continue + _verify_snapshot_integrity(input_snapshots, "train") + _validate_eval_result( + train_result, + expected_prompt_id=candidate.candidate_id, + expected_split="train", + ) + record["train_result"] = train_result + explicit_evaluator_cost = round(explicit_evaluator_cost + train_result.cost, 6) + _verify_snapshot_integrity(input_snapshots, "validation") + validation_artifact_dir = _artifact_dir( + run_root, + "evaluations", + path_label, + "validation", + ) + try: + validation_result = await _await_backend_with_prompt_integrity( + lambda: evaluator.evaluate( + prompt_id=candidate.candidate_id, + prompts=dict(bundle), + dataset_path=validation_path, + split="validation", + trace=request.trace, + artifact_dir=validation_artifact_dir, + ), + prompt_snapshot=prompt_snapshot, + context=f"candidate {candidate.candidate_id!r} validation evaluation", + ) + except Exception as error: + record["evaluation_error"] = _recoverable_candidate_evaluation_failure( + error, + stage="validation", + completed_results=[train_result], + input_snapshots=input_snapshots, + prompt_snapshot=prompt_snapshot, + ) + candidate_records.append(record) + candidate_evaluation_failed = True + all_results_comparable = False + continue + _verify_snapshot_integrity(input_snapshots, "validation") + _validate_eval_result( + validation_result, + expected_prompt_id=candidate.candidate_id, + expected_split="validation", + ) + record["validation_result"] = validation_result + explicit_evaluator_cost = round(explicit_evaluator_cost + validation_result.cost, 6) + candidate_records.append(record) + + run_cost_complete = optimization.cost.complete and not candidate_evaluation_failed + gate_cost_summary = replace(optimization.cost, complete=run_cost_complete) + cumulative_cost = round(baseline_evaluator_cost + optimization.cost.total, 6) + for record in candidate_records: + candidate = record["candidate"] + if "evaluation_error" in record: + decision = _evaluation_error_gate_decision( + candidate_id=candidate.candidate_id, + failure=record["evaluation_error"], + cumulative_cost=cumulative_cost, + ) + gate_decisions.append(decision) + cumulative_cost = decision.total_run_cost + continue + + train_result = record["train_result"] + validation_result = record["validation_result"] + comparable = _result_pair_is_comparable(baseline_train, train_result) and (_result_pair_is_comparable( + baseline_validation, validation_result)) + all_results_comparable = all_results_comparable and comparable + deltas = (compute_case_deltas( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + ) if comparable else []) + decision = gate.decide( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + deltas=deltas, + cost_summary=gate_cost_summary, + cumulative_cost=cumulative_cost, + ) + all_deltas.extend(deltas) + gate_decisions.append(decision) + cumulative_cost = decision.total_run_cost + + selected_candidate = _select_candidate(candidate_records, gate_decisions) + cost_summary = CostSummary( + optimizer=optimization.cost.optimizer, + evaluator=round(optimization.cost.evaluator + explicit_evaluator_cost, 6), + agent=optimization.cost.agent, + total=cumulative_cost, + complete=run_cost_complete, + reported_optimizer_cost=_reported_optimizer_cost(optimization.cost), + ) + _validate_cost_summary(cost_summary, context="pipeline cost summary") + input_hashes: dict[str, Any] = { + **input_snapshots.hashes(), + "target_prompts": prompt_snapshot.hashes(), + } + before_hashes = prompt_snapshot.hashes() + writeback_journal = _initial_writeback_journal( + run_id=run_id, + selected_candidate=selected_candidate, + update_source=request.update_source, + before_hashes=before_hashes, + candidate_hashes=(_prompt_bundle_hashes(candidate_bundles[selected_candidate]) + if selected_candidate is not None else {}), + input_hashes=input_snapshots.hashes(), + ) + audit = _build_audit( + request=request, + started=started, + snapshot_hashes=prompt_snapshot.hashes(), + input_hashes=input_hashes, + config_snapshot=_sanitize_public_value(raw_config), + raw_config_hash=stable_config_hash(raw_config), + effective_seed=effective_seed, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + candidate_bundles=candidate_bundles, + optimization_raw_summary=optimization.raw_summary, + optimization_cost=optimization.cost, + cost_summary=cost_summary, + all_results_comparable=all_results_comparable, + effective_gate_config=effective_gate_config, + writeback_journal=writeback_journal, + ) + run = _build_run(request, baseline_train=baseline_train, baseline_validation=baseline_validation) + if selected_candidate is None: + provisional_writeback = WritebackResult(status="rejected", before_hashes=before_hashes) + else: + provisional_writeback = WritebackResult( + status="not_requested", + before_hashes=before_hashes, + error=("pending source writeback; pre-write audit must be prepared before commit" + if request.update_source else None), + ) + report = build_report( + run=run, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + per_case_deltas=all_deltas, + gate_decisions=gate_decisions, + selected_candidate=selected_candidate, + audit=audit, + baseline_prompts=dict(baseline_prompts), + rounds=optimization.rounds, + cost_summary=cost_summary, + writeback=provisional_writeback, + ) + + _verify_prompt_integrity( + prompt_snapshot, + context="immediately before preparing run artifacts", + ) + prepare_run_artifacts(report, artifact_paths) + if selected_candidate is None or not request.update_source: + terminal_journal = _terminal_journal( + writeback_journal, + provisional_writeback, + state=provisional_writeback.status, + ) + final_report = _with_writeback(report, terminal_journal, provisional_writeback) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="before persisting no-write outcome", + artifact_paths=artifact_paths, + journal=terminal_journal, + request=request, + ) + persist_writeback_outcome(final_report, artifact_paths) + finalize_run_artifacts( + final_report, + artifact_paths, + before_publish=lambda: _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="immediately before publishing no-write run", + artifact_paths=artifact_paths, + journal=terminal_journal, + request=request, + ), + ) + return final_report + + input_drift = _public_input_drift(_input_drift(input_snapshots), request) + if input_drift: + error = "input changed after pipeline entry: " + ", ".join(sorted(input_drift)) + writeback = WritebackResult( + status="rejected", + before_hashes=before_hashes, + after_hashes=_current_prompt_hashes(prompt_snapshot), + error=error, + ) + final_journal = _terminal_journal( + writeback_journal, + writeback, + state="conflict", + input_drift=input_drift, + ) + final_report = _with_writeback(report, final_journal, writeback, input_drift=input_drift) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="before persisting input-drift outcome", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ) + persist_writeback_outcome(final_report, artifact_paths) + finalize_run_artifacts( + final_report, + artifact_paths, + before_publish=lambda: _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=prompt_snapshot.hashes(), + context="immediately before publishing input-drift run", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ), + ) + return final_report + + committing_journal = dict(writeback_journal) + committing_journal.update({"state": "committing", "report_phase": "writeback"}) + persist_writeback_journal(artifact_paths, committing_journal) + try: + writeback = commit_prompt_bundle( + prompt_snapshot, + dict(candidate_bundles[selected_candidate]), + ) + except ConcurrentPromptUpdateError as error: + writeback = WritebackResult( + status="rejected", + before_hashes=before_hashes, + after_hashes=_current_prompt_hashes(prompt_snapshot), + error=_sanitize_error_message( + f"prompt writeback conflict: {error}", + request, + ), + ) + terminal_state = "conflict" + else: + writeback = _sanitize_writeback_result(writeback, request) + terminal_state = writeback.status + terminal_prompt_hashes = dict(writeback.after_hashes or prompt_snapshot.hashes()) + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=terminal_prompt_hashes, + context="immediately after source writeback", + artifact_paths=artifact_paths, + journal=committing_journal, + request=request, + ) + final_journal = _terminal_journal( + committing_journal, + writeback, + state=terminal_state, + ) + final_report = _with_writeback(report, final_journal, writeback) + try: + persist_writeback_journal(artifact_paths, final_journal) + except Exception as persist_error: + unknown_journal = dict(final_journal) + unknown_journal.update({ + "state": + "unknown", + "error": + _sanitize_error_message( + f"failed to persist terminal writeback outcome: {persist_error}", + request, + ), + "observed_hashes": + _current_prompt_hashes(prompt_snapshot), + }) + try: + persist_writeback_journal(artifact_paths, unknown_journal) + except Exception: + pass + raise + _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=terminal_prompt_hashes, + context="after persisting terminal writeback journal", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ) + persist_writeback_outcome(final_report, artifact_paths) + finalize_run_artifacts( + final_report, + artifact_paths, + before_publish=lambda: _verify_terminal_writeback_integrity( + prompt_snapshot, + expected_hashes=terminal_prompt_hashes, + context="immediately before publishing writeback run", + artifact_paths=artifact_paths, + journal=final_journal, + request=request, + ), + ) + return final_report + + +def _snapshot_pipeline_inputs(request: PipelineRequest, *, run_id: str) -> _InputSnapshots: + sources: dict[str, Path] = { + "train": Path(request.train_path), + "validation": Path(request.validation_path), + "optimizer": Path(request.optimizer_config_path), + } + if request.gate_config_path is not None: + sources["gate_config"] = Path(request.gate_config_path) + snapshots: dict[str, _InputFileSnapshot] = {} + try: + for role, source in sources.items(): + content = source.read_bytes() + fd, temp_name = tempfile.mkstemp( + dir=source.parent, + prefix=f".{run_id}.{role}.", + suffix=f".snapshot-{source.name}", + ) + snapshot_path = Path(temp_name) + try: + with os.fdopen(fd, "wb") as stream: + fd = -1 + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + except BaseException as primary_error: + try: + snapshot_path.unlink() + except FileNotFoundError: + pass + except OSError as cleanup_error: + add_note = getattr(primary_error, "add_note", None) + if add_note is not None: + add_note(f"failed to remove partial input snapshot: {cleanup_error}") + raise + finally: + if fd >= 0: + os.close(fd) + snapshots[role] = _InputFileSnapshot( + role=role, + original_path=source, + snapshot_path=snapshot_path, + sha256=hashlib.sha256(content).hexdigest(), + ) + except BaseException as primary_error: + try: + _InputSnapshots(snapshots).cleanup() + except OSError as cleanup_error: + add_note = getattr(primary_error, "add_note", None) + if add_note is not None: + add_note(str(cleanup_error)) + raise + return _InputSnapshots(snapshots) + + +def _verify_snapshot_integrity( + snapshots: _InputSnapshots, + *requested_roles: str, +) -> None: + roles = requested_roles or tuple(snapshots.files) + for role in roles: + item = snapshots.files[role] + try: + observed = hashlib.sha256(item.snapshot_path.read_bytes()).hexdigest() + except OSError as error: + raise ValueError(f"immutable input snapshot {role!r} is unavailable: {error}") from error + if observed != item.sha256: + raise ValueError(f"immutable input snapshot changed for {role}") + + +def _effective_gate_config( + request: PipelineRequest, + snapshots: _InputSnapshots, + *, + raw_config: dict[str, Any], +) -> dict[str, Any]: + if "gate_config" in snapshots.files: + payload = read_json(snapshots.path("gate_config")) + gate_payload = payload.get("gate", payload) + if gate_payload is None: + gate_payload = {} + if not isinstance(gate_payload, dict): + raise ValueError("gate config field 'gate' must be an object") + elif request.gate_config_source == "optimizer": + gate_payload = parse_optimizer_config( + raw_config, + path=request.optimizer_config_path, + ).gate.to_dict() + elif request.gate_config_source == "request": + gate_payload = request.gate_config + else: + raise ValueError(f"unknown gate_config_source: {request.gate_config_source!r}") + if not isinstance(gate_payload, dict): + raise ValueError("gate config must be an object") + return _parse_gate_config(gate_payload, path="gate config snapshot").to_dict() + + +def _validate_backend_seed(backend: Any, effective_seed: int) -> None: + if not hasattr(backend, "seed"): + return + backend_seed = getattr(backend, "seed") + if (isinstance(backend_seed, bool) or not isinstance(backend_seed, int) or backend_seed != effective_seed): + raise ValueError(f"fake backend seed {backend_seed!r} does not match effective seed {effective_seed}") + + +def _strict_case_metadata(path: str | Path, *, role: str) -> dict[str, bool]: + payload = read_json(path) + has_standard = "eval_cases" in payload + has_legacy = "cases" in payload + if has_standard == has_legacy: + raise ValueError(f"{role} evalset must contain exactly one of eval_cases or cases") + if has_standard: + cases = payload["eval_cases"] + standard = True + else: + cases = payload["cases"] + standard = False + if not isinstance(cases, list): + raise ValueError(f"{role} evalset cases must be a list") + if not cases: + raise ValueError(f"{role} evalset cases must not be empty") + + metadata: dict[str, bool] = {} + for index, case in enumerate(cases): + if not isinstance(case, dict): + raise ValueError(f"{role} evalset case {index} must be an object") + if standard: + case_id = case.get("eval_id") + protected = False + if "session_input" in case: + session_input = case["session_input"] + if not isinstance(session_input, dict): + raise ValueError(f"{role} evalset case {index} session_input must be an object") + state = session_input.get("state", {}) + if state is None: + state = {} + if not isinstance(state, dict): + raise ValueError(f"{role} evalset case {index} session_input.state must be an object") + protected = state.get("eval_optimize_protected", False) + else: + case_id = case["case_id"] if "case_id" in case else case.get("id") + protected = case.get("protected", False) + + if not isinstance(case_id, str) or not case_id.strip(): + raise ValueError(f"{role} evalset case {index} case ID must be a non-empty string") + if not isinstance(protected, bool): + raise ValueError(f"{role} evalset case {case_id!r} protected metadata must be a boolean") + if case_id in metadata: + raise ValueError(f"{role} evalset contains duplicate case ID {case_id!r}") + metadata[case_id] = protected + return metadata + + +def _input_drift(snapshots: _InputSnapshots) -> dict[str, dict[str, Any]]: + drift: dict[str, dict[str, Any]] = {} + for role, item in snapshots.files.items(): + try: + after_hash = hashlib.sha256(item.original_path.read_bytes()).hexdigest() + except OSError as error: + drift[role] = { + "before_hash": item.sha256, + "after_hash": None, + "error": str(error), + } + continue + if after_hash != item.sha256: + drift[role] = { + "before_hash": item.sha256, + "after_hash": after_hash, + "error": None, + } + return drift + + +def _public_input_drift( + drift: dict[str, dict[str, Any]], + request: PipelineRequest, +) -> dict[str, dict[str, Any]]: + public: dict[str, dict[str, Any]] = {} + for role, details in drift.items(): + sanitized = dict(details) + if sanitized.get("error"): + sanitized["error"] = _sanitize_error_message(str(sanitized["error"]), request) + public[role] = sanitized + return public + + +def _prompt_bundle_hashes(bundle: Mapping[str, str]) -> dict[str, str]: + return {name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() for name, prompt in bundle.items()} + + +def _current_prompt_hashes(snapshot: Any) -> dict[str, str]: + hashes: dict[str, str] = {} + for name, prompt_file in snapshot.files.items(): + try: + hashes[name] = hashlib.sha256(prompt_file.path.read_bytes()).hexdigest() + except OSError: + continue + return hashes + + +def _verify_prompt_integrity( + snapshot: Any, + *, + context: str, + expected_hashes: Mapping[str, str] | None = None, +) -> None: + expected = dict(expected_hashes or snapshot.hashes()) + observed: dict[str, str] = {} + for name, prompt_file in snapshot.files.items(): + try: + observed[name] = hashlib.sha256(prompt_file.path.read_bytes()).hexdigest() + except OSError as error: + raise PromptRestorationError(f"source prompt field {name!r} is unreadable {context}") from error + if observed != expected: + changed = sorted(name for name in set(expected) | set(observed) if expected.get(name) != observed.get(name)) + raise ConcurrentPromptUpdateError(f"source prompt integrity changed {context}: {', '.join(changed)}") + + +async def _await_backend_with_prompt_integrity( + operation: Callable[[], Awaitable[Any]], + *, + prompt_snapshot: Any, + context: str, +) -> Any: + """Run one backend await while making source-prompt drift run-fatal.""" + + _verify_prompt_integrity( + prompt_snapshot, + context=f"before {context}", + ) + try: + result = await operation() + except BaseException as backend_error: + try: + _verify_prompt_integrity( + prompt_snapshot, + context=f"after failed {context}", + ) + except (ConcurrentPromptUpdateError, PromptRestorationError) as integrity_error: + raise integrity_error from backend_error + raise + _verify_prompt_integrity( + prompt_snapshot, + context=f"after {context}", + ) + return result + + +def _verify_terminal_writeback_integrity( + snapshot: Any, + *, + expected_hashes: Mapping[str, str], + context: str, + artifact_paths: Any, + journal: dict[str, Any], + request: PipelineRequest, +) -> None: + try: + _verify_prompt_integrity( + snapshot, + context=context, + expected_hashes=expected_hashes, + ) + except (ConcurrentPromptUpdateError, PromptRestorationError) as integrity_error: + unknown_journal = dict(journal) + unknown_journal.update({ + "state": "unknown", + "report_phase": "final", + "after_hashes": _current_prompt_hashes(snapshot), + "error": _sanitize_error_message(str(integrity_error), request), + }) + try: + persist_writeback_journal(artifact_paths, unknown_journal) + except Exception as persist_error: + add_note = getattr(integrity_error, "add_note", None) + if add_note is not None: + add_note(f"failed to persist unknown writeback state: {persist_error}") + raise + + +def _terminal_journal( + journal: dict[str, Any], + writeback: WritebackResult, + *, + state: str, + input_drift: dict[str, Any] | None = None, +) -> dict[str, Any]: + terminal = dict(journal) + terminal.update({ + "state": state, + "report_phase": "final", + "after_hashes": dict(writeback.after_hashes), + "error": writeback.error, + }) + if input_drift: + terminal["input_drift"] = input_drift + return terminal + + +def _with_writeback( + report: OptimizationReport, + journal: dict[str, Any], + writeback: WritebackResult, + *, + input_drift: dict[str, Any] | None = None, +) -> OptimizationReport: + audit = dict(report.audit) + audit["writeback_journal"] = journal + if input_drift: + audit["input_drift"] = input_drift + return replace(report, audit=audit, writeback=writeback) + + +def _sanitize_writeback_result( + writeback: WritebackResult, + request: PipelineRequest, +) -> WritebackResult: + if writeback.error is None: + return writeback + return replace( + writeback, + error=_sanitize_error_message(writeback.error, request), + ) + + +def _sanitize_error_message(message: str, request: PipelineRequest) -> str: + replacements: list[tuple[str, str]] = [] + raw_paths: list[str | Path] = [ + request.train_path, + request.validation_path, + request.optimizer_config_path, + request.output_dir, + *request.target_prompt_paths.values(), + ] + if request.gate_config_path is not None: + raw_paths.append(request.gate_config_path) + for raw_path in raw_paths: + path = Path(raw_path) + display = _display_path(path) + replacements.append((str(path), display)) + replacements.append((str(path.resolve()), display)) + sanitized = message + for source, replacement in sorted(replacements, key=lambda item: len(item[0]), reverse=True): + sanitized = sanitized.replace(source, replacement) + return sanitized + + +def _decode_snapshot(snapshot: Any) -> dict[str, str]: + prompts: dict[str, str] = {} + for name, prompt_file in snapshot.files.items(): + try: + prompts[name] = prompt_file.content.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise ValueError(f"target prompt {name!r} at {prompt_file.path} is not valid UTF-8") from exc + return prompts + + +def _recoverable_candidate_evaluation_failure( + error: Exception, + *, + stage: str, + completed_results: list[EvalResult], + input_snapshots: _InputSnapshots, + prompt_snapshot: Any, +) -> dict[str, Any]: + """Return a public failure record only after proving prompt state is safe. + + Candidate backend errors are recoverable, but source-prompt restoration, + CAS, immutable-input, and cancellation failures remain run-fatal. + """ + + _verify_snapshot_integrity(input_snapshots) + if _exception_chain_contains(error, (ConcurrentPromptUpdateError, PromptRestorationError)): + raise error + try: + _verify_prompt_integrity( + prompt_snapshot, + context="after failed candidate evaluation", + ) + except (ConcurrentPromptUpdateError, PromptRestorationError) as integrity_error: + raise integrity_error from error + + raw_message = str(error) + return { + "stage": stage, + "type": type(error).__name__, + "message": "candidate evaluation failed; backend details withheld", + "message_sha256": hashlib.sha256(raw_message.encode("utf-8", errors="replace")).hexdigest(), + "completed_splits": [result.split for result in completed_results], + "known_evaluator_cost": round(sum(result.cost for result in completed_results), 6), + "cost_complete": False, + } + + +def _exception_chain_contains( + error: BaseException, + exception_types: tuple[type[BaseException], ...], +) -> bool: + pending: list[BaseException] = [error] + seen: set[int] = set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, exception_types): + return True + if current.__cause__ is not None: + pending.append(current.__cause__) + if current.__context__ is not None: + pending.append(current.__context__) + return False + + +def _evaluation_error_gate_decision( + *, + candidate_id: str, + failure: dict[str, Any], + cumulative_cost: float, +) -> GateDecision: + candidate_cost = round(float(failure["known_evaluator_cost"]), 6) + total_run_cost = round(cumulative_cost + candidate_cost, 6) + stage = str(failure["stage"]) + reason = (f"reject: evaluation_error during {stage}: " + f"{failure['type']}: {failure['message']}") + return GateDecision( + candidate_id=candidate_id, + accepted=False, + reasons=[reason], + train_score_delta=None, + validation_score_delta=None, + new_hard_failures=[], + protected_regressions=[], + validation_new_failures=[], + excessive_score_drops=[], + overfit_detected=None, + candidate_cost=candidate_cost, + cumulative_cost=round(cumulative_cost, 6), + total_run_cost=total_run_cost, + cost=candidate_cost, + gate_status="not_applied", + gate_not_applied_reason="evaluation_error", + not_applied_checks=[ + "score_delta", + "validation_improvement", + "new_failures", + "hard_failures", + "protected_regressions", + "per_case_score_drop", + "overfit", + ], + ) + + +def _validated_candidate_bundles( + candidates: list[CandidatePrompt], + *, + expected_fields: set[str], +) -> dict[str, Mapping[str, str]]: + bundles: dict[str, Mapping[str, str]] = {} + casefold_ids: set[str] = set() + for candidate in candidates: + candidate_id = candidate.candidate_id + _validate_candidate_id(candidate_id) + for field_name, value in ( + ("prompt", candidate.prompt), + ("rationale", candidate.rationale), + ("prompt_diff", candidate.prompt_diff), + ): + _validate_utf8_text( + value, + context=f"candidate {candidate_id!r} {field_name}", + ) + if candidate_id.casefold() == "baseline": + raise ValueError("candidate_id 'baseline' is reserved") + if candidate_id in bundles: + raise ValueError(f"duplicate candidate_id: {candidate_id}") + if candidate_id.casefold() in casefold_ids: + raise ValueError(f"candidate_id values must be case-insensitively unique: {candidate_id}") + casefold_ids.add(candidate_id.casefold()) + bundle = candidate.bundle() + actual_fields = set(bundle) + if actual_fields != expected_fields: + raise ValueError(f"candidate {candidate_id!r} bundle fields must exactly match target prompt fields; " + f"missing={sorted(expected_fields - actual_fields)}, " + f"extra={sorted(actual_fields - expected_fields)}") + for field_name, prompt_text in bundle.items(): + if not isinstance(prompt_text, str) or not prompt_text: + raise ValueError(f"candidate {candidate_id!r} bundle field {field_name!r} " + "must be a non-empty string") + _validate_utf8_text( + prompt_text, + context=f"candidate {candidate_id!r} bundle field {field_name!r}", + ) + bundles[candidate_id] = MappingProxyType(dict(bundle)) + return bundles + + +def _validate_candidate_id(candidate_id: Any) -> None: + try: + validate_artifact_component(candidate_id, context="candidate_id") + except ValueError as error: + raise ValueError(f"candidate_id is not artifact-safe: {candidate_id!r}") from error + + +def _validate_utf8_text(value: Any, *, context: str) -> None: + if not isinstance(value, str): + raise ValueError(f"{context} must be a string") + try: + value.encode("utf-8", errors="strict") + except UnicodeEncodeError as error: + raise ValueError(f"{context} must be valid UTF-8") from error + + +def _result_pair_is_comparable(baseline: EvalResult, candidate: EvalResult) -> bool: + baseline_ids = [case.case_id for case in baseline.cases] + candidate_ids = [case.case_id for case in candidate.cases] + return (bool(baseline_ids) and bool(candidate_ids) + and (len(baseline_ids) == len(set(baseline_ids)) and len(candidate_ids) == len(set(candidate_ids)) + and set(baseline_ids) == set(candidate_ids))) + + +def _validate_eval_result( + result: EvalResult, + *, + expected_prompt_id: str, + expected_split: str, + expected_case_ids: set[str] | None = None, +) -> None: + if result.prompt_id != expected_prompt_id: + raise ValueError(f"evaluation result prompt_id mismatch: expected {expected_prompt_id!r}, " + f"got {result.prompt_id!r}") + if result.split != expected_split: + raise ValueError(f"EvalResult split mismatch for {expected_prompt_id!r}: " + f"expected {expected_split!r}, got {result.split!r}") + score = _finite_number( + result.score, + context=f"EvalResult score for {expected_prompt_id!r}", + minimum=0.0, + maximum=1.0, + ) + _finite_number( + result.cost, + context=f"EvalResult cost for {expected_prompt_id!r}", + minimum=0.0, + ) + if not isinstance(result.passed, bool): + raise ValueError(f"EvalResult passed for {expected_prompt_id!r} must be a boolean") + case_ids: set[str] = set() + for case in result.cases: + if not isinstance(case.case_id, str) or not case.case_id: + raise ValueError("CaseResult case_id must be a non-empty string") + if case.case_id in case_ids: + # Duplicate IDs are handled by the fail-closed gate, but the numeric + # protocol boundary still validates each record independently. + pass + case_ids.add(case.case_id) + if case.split != expected_split: + raise ValueError(f"CaseResult {case.case_id!r} split mismatch for {expected_prompt_id!r}: " + f"expected {expected_split!r}, got {case.split!r}") + _finite_number( + case.score, + context=f"CaseResult {case.case_id!r} score", + minimum=0.0, + maximum=1.0, + ) + _finite_number( + case.cost, + context=f"CaseResult {case.case_id!r} cost", + minimum=0.0, + ) + if not isinstance(case.passed, bool) or not isinstance(case.hard_failed, bool): + raise ValueError(f"CaseResult {case.case_id!r} passed/hard_failed must be booleans") + if not isinstance(case.metrics, dict): + raise ValueError(f"CaseResult {case.case_id!r} metrics must be an object") + for metric_name, metric_value in case.metrics.items(): + if not isinstance(metric_name, str) or not metric_name: + raise ValueError(f"CaseResult {case.case_id!r} metric names must be strings") + _finite_number( + metric_value, + context=f"CaseResult {case.case_id!r} metric {metric_name!r}", + ) + if result.cases: + mean_score = sum(float(case.score) for case in result.cases) / len(result.cases) + if not math.isclose(score, mean_score, rel_tol=0.0, abs_tol=1e-6): + raise ValueError(f"EvalResult score for {expected_prompt_id!r} must equal the case mean") + expected_passed = bool(result.cases) and all(case.passed for case in result.cases) + if result.passed != expected_passed: + raise ValueError(f"EvalResult passed for {expected_prompt_id!r} must agree with case results") + if expected_case_ids is not None: + observed_ids = [case.case_id for case in result.cases] + if len(observed_ids) != len(set(observed_ids)) or set(observed_ids) != expected_case_ids: + raise ValueError(f"{expected_prompt_id} {expected_split} result must exactly match dataset case IDs; " + f"expected={sorted(expected_case_ids)}, observed={sorted(set(observed_ids))}") + + +def _validate_optimization_result(result: OptimizationResult) -> None: + if not isinstance(result.candidates, list) or not isinstance(result.rounds, list): + raise ValueError("optimization result candidates and rounds must be lists") + _validate_cost_summary(result.cost, context="optimization cost") + for index, round_record in enumerate(result.rounds): + if (isinstance(round_record.round_id, bool) or not isinstance(round_record.round_id, int) + or round_record.round_id <= 0): + raise ValueError(f"optimization round {index} round_id must be a positive integer") + _validate_candidate_id(round_record.candidate_id) + _finite_number( + round_record.duration_seconds, + context=f"optimization round {round_record.round_id} duration", + minimum=0.0, + ) + if not isinstance(round_record.metrics, dict): + raise ValueError(f"optimization round {round_record.round_id} metrics must be an object") + for metric_name, metric_value in round_record.metrics.items(): + _finite_number( + metric_value, + context=f"optimization round {round_record.round_id} metric {metric_name!r}", + ) + _validate_cost_summary( + round_record.cost, + context=f"optimization round {round_record.round_id} cost", + ) + validate_unique_round_ids(result.rounds) + if not isinstance(result.raw_summary, dict): + raise ValueError("optimization raw_summary must be an object") + _validate_nested_numbers(result.raw_summary, context="optimization raw_summary") + + +def _validate_cost_summary(summary: CostSummary, *, context: str) -> None: + components = { + "optimizer": _finite_number(summary.optimizer, context=f"{context} optimizer", minimum=0.0), + "evaluator": _finite_number(summary.evaluator, context=f"{context} evaluator", minimum=0.0), + "agent": _finite_number(summary.agent, context=f"{context} agent", minimum=0.0), + } + total = _finite_number(summary.total, context=f"{context} total", minimum=0.0) + if not isinstance(summary.complete, bool): + raise ValueError(f"{context} complete must be a boolean") + if summary.reported_optimizer_cost is not None: + _finite_number( + summary.reported_optimizer_cost, + context=f"{context} reported_optimizer_cost", + minimum=0.0, + ) + if summary.complete and not math.isclose( + total, + sum(components.values()), + rel_tol=0.0, + abs_tol=1e-6, + ): + raise ValueError(f"{context} total must equal components when complete") + + +def _reported_optimizer_cost(summary: CostSummary) -> float | None: + if summary.complete: + return None + if summary.reported_optimizer_cost is not None: + return float(summary.reported_optimizer_cost) + return float(summary.total) + + +def _validate_nested_numbers(value: Any, *, context: str) -> None: + if isinstance(value, bool) or value is None or isinstance(value, str): + return + if isinstance(value, (int, float)): + _finite_number(value, context=context) + return + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"{context} keys must be JSON-compatible strings") + _validate_nested_numbers(item, context=f"{context}.{key}") + return + if isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _validate_nested_numbers(item, context=f"{context}[{index}]") + return + raise ValueError(f"{context} contains non JSON-compatible value {type(value).__name__}") + + +def _finite_number( + value: Any, + *, + context: str, + minimum: float | None = None, + maximum: float | None = None, +) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{context} must be a finite number") + try: + number = float(value) + except OverflowError as error: + raise ValueError(f"{context} must be a finite number") from error + if not math.isfinite(number): + raise ValueError(f"{context} must be a finite number") + if minimum is not None and number < minimum: + raise ValueError(f"{context} must be non-negative" if minimum == 0 else f"{context} is too small") + if maximum is not None and number > maximum: + raise ValueError(f"{context} must be between {minimum:g} and {maximum:g}") + return number + + +def _gate_config_with_dataset_protection( + gate_config: dict[str, Any], + *, + validation_metadata: dict[str, bool], +) -> dict[str, Any]: + effective = dict(gate_config) + configured_ids = effective.get("protected_case_ids", []) + if not isinstance(configured_ids, list) or not all( + isinstance(case_id, str) and bool(case_id.strip()) for case_id in configured_ids): + raise ValueError("gate protected_case_ids must be a list of non-empty strings") + missing_ids = sorted(set(configured_ids) - set(validation_metadata)) + if missing_ids: + raise ValueError("gate protected_case_ids references missing validation cases: " + ", ".join(missing_ids)) + protected_ids = set(configured_ids) + protected_ids.update(case_id for case_id, protected in validation_metadata.items() if protected) + effective["protected_case_ids"] = sorted(protected_ids) + return effective + + +def _select_candidate( + candidate_records: list[dict[str, Any]], + gate_decisions: list[Any], +) -> str | None: + decisions_by_id = {decision.candidate_id: decision for decision in gate_decisions} + accepted: list[tuple[int, dict[str, Any]]] = [] + for index, record in enumerate(candidate_records): + candidate = record["candidate"] + decision = decisions_by_id[candidate.candidate_id] + if (decision.accepted and "train_result" in record and "validation_result" in record): + accepted.append((index, record)) + if not accepted: + return None + _, selected = max( + accepted, + key=lambda item: ( + item[1]["validation_result"].score, + item[1]["train_result"].score, + -item[0], + ), + ) + return selected["candidate"].candidate_id + + +def _build_run( + request: PipelineRequest, + *, + baseline_train: EvalResult, + baseline_validation: EvalResult, +) -> dict[str, Any]: + target_paths = {name: _display_path(path) for name, path in request.target_prompt_paths.items()} + return { + "run_id": request.run_id, + "mode": request.mode, + "trace": request.trace, + "trace_enabled": request.trace, + "update_source": request.update_source, + "train_cases": len(baseline_train.cases), + "validation_cases": len(baseline_validation.cases), + "target_prompt_paths": target_paths, + "prompt_source": target_paths.get("system_prompt"), + "paths": { + "train": _display_path(request.train_path), + "validation": _display_path(request.validation_path), + "optimizer": _display_path(request.optimizer_config_path), + "prompts": target_paths, + }, + "reproducibility_shell": "powershell", + "reproducibility_command": _reproducibility_command(request), + } + + +def _build_audit( + *, + request: PipelineRequest, + started: float, + snapshot_hashes: dict[str, str], + input_hashes: dict[str, Any], + config_snapshot: dict[str, Any], + raw_config_hash: str, + effective_seed: int, + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_records: list[dict[str, Any]], + candidate_bundles: dict[str, Mapping[str, str]], + optimization_raw_summary: dict[str, Any], + optimization_cost: CostSummary, + cost_summary: CostSummary, + all_results_comparable: bool, + effective_gate_config: dict[str, Any], + writeback_journal: dict[str, Any], +) -> dict[str, Any]: + candidate_costs = { + record["candidate"].candidate_id: + round( + sum(record[result_name].cost for result_name in ("train_result", "validation_result") + if result_name in record), + 6, + ) + for record in candidate_records + } + candidate_evaluation_failures = { + record["candidate"].candidate_id: dict(record["evaluation_error"]) + for record in candidate_records if "evaluation_error" in record + } + candidate_prompt_hashes = { + candidate_id: { + name: hashlib.sha256(prompt.encode("utf-8")).hexdigest() + for name, prompt in bundle.items() + } + for candidate_id, bundle in candidate_bundles.items() + } + cost_audit: dict[str, Any] = { + "baseline": round(baseline_train.cost + baseline_validation.cost, 6), + "candidates": candidate_costs, + "optimization": to_jsonable(optimization_cost), + "evaluator": cost_summary.evaluator, + "total": cost_summary.total if cost_summary.complete else None, + "complete": cost_summary.complete, + "reported_optimizer_cost": cost_summary.reported_optimizer_cost, + } + if not cost_summary.complete: + cost_audit["known_run_cost"] = cost_summary.total + duration = max(time.perf_counter() - started, 1e-9) + target_paths = {name: _display_path(path) for name, path in request.target_prompt_paths.items()} + return { + "seed": effective_seed, + "duration_seconds": duration, + "config_hash": raw_config_hash, + "config_file_sha256": input_hashes["optimizer"], + "redacted_config_hash": stable_config_hash(config_snapshot), + "redacted_config_snapshot_sha256": _pretty_json_sha256(config_snapshot), + "config_snapshot": config_snapshot, + "gate_config_hash": stable_config_hash(effective_gate_config), + "gate_config_snapshot": effective_gate_config, + "writeback_journal": writeback_journal, + "input_hashes": input_hashes, + "input_paths": { + "train": _display_path(request.train_path), + "validation": _display_path(request.validation_path), + "optimizer": _display_path(request.optimizer_config_path), + "prompts": target_paths, + **({ + "prompt": target_paths["system_prompt"] + } if "system_prompt" in target_paths else {}), + }, + "prompt_hash": snapshot_hashes.get("system_prompt"), + "prompt_hashes": snapshot_hashes, + "candidate_prompt_hashes": candidate_prompt_hashes, + "candidate_artifacts": { + record["candidate"].candidate_id: _candidate_artifact_component( + index, + record["candidate"].candidate_id, + ) + for index, record in enumerate(candidate_records, start=1) + }, + "candidate_prompts": { + candidate_id: dict(bundle) + for candidate_id, bundle in candidate_bundles.items() + }, + "candidate_evaluation_failures": candidate_evaluation_failures, + "prompt_diffs": { + record["candidate"].candidate_id: record["candidate"].prompt_diff + for record in candidate_records + }, + "total_run_cost": cost_summary.total if cost_summary.complete else None, + "known_run_cost": cost_summary.total if not cost_summary.complete else None, + "total_run_cost_complete": cost_summary.complete, + "cost": cost_audit, + "sdk_result_summary": _sanitize_public_value(to_jsonable(optimization_raw_summary)), + "sdk_result_availability": { + "aggregate_validation_result": + bool(request.mode == "sdk" and optimization_raw_summary), + "full_train_eval_result": + not any(failure.get("stage") == "train" for failure in candidate_evaluation_failures.values()), + "full_per_case_validation_delta": + all_results_comparable, + }, + "reproducibility_shell": "powershell", + "reproducibility_command": _reproducibility_command(request), + } + + +def _artifact_dir(root: Path, *parts: str) -> Path: + path = root.joinpath(*parts) + path.mkdir(parents=True, exist_ok=True) + return path + + +def _safe_artifact_component(value: str, *, context: str) -> str: + return validate_artifact_component(value, context=context) + + +def _candidate_artifact_component(index: int, candidate_id: str) -> str: + digest = hashlib.sha256(candidate_id.encode("utf-8")).hexdigest()[:12] + return f"{index:03d}-{digest}" + + +def _display_path(raw_path: str | Path) -> str: + path = Path(raw_path).resolve() + try: + relative = path.relative_to(_REPOSITORY_ROOT) + except ValueError: + return f"$EXTERNAL/{path.name}" + return relative.as_posix() + + +def _redact_secrets(value: Any) -> Any: + if isinstance(value, dict): + redacted: dict[str, Any] = {} + for key, item in value.items(): + key_text = str(key) + lowered = key_text.casefold() + if _is_secret_key(lowered): + redacted[key_text] = "" + else: + redacted[key_text] = _redact_secrets(item) + return redacted + if isinstance(value, list): + return [_redact_secrets(item) for item in value] + if isinstance(value, tuple): + return [_redact_secrets(item) for item in value] + return value + + +def _pretty_json_sha256(value: Any) -> str: + content = json.dumps( + value, + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + "\n" + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _is_secret_key(lowered: str) -> bool: + normalized = "".join(character for character in lowered if character.isalnum()) + return normalized.endswith(_SENSITIVE_CONFIG_KEY_SUFFIXES) or any(marker in normalized + for marker in _SENSITIVE_CONFIG_KEY_MARKERS) + + +def _sanitize_public_value(value: Any) -> Any: + value = _redact_secrets(value) + if isinstance(value, dict): + return {str(key): _sanitize_public_value(item) for key, item in value.items()} + if isinstance(value, list): + return [_sanitize_public_value(item) for item in value] + if isinstance(value, str): + try: + candidate = Path(value) + if candidate.is_absolute(): + return _display_path(candidate) + except (OSError, ValueError): + pass + return value + + +def _reproducibility_command(request: PipelineRequest) -> str: + command = [ + "python", + "examples/optimization/eval_optimize_loop/run_pipeline.py", + "--mode", + request.mode, + "--train", + _display_path(request.train_path), + "--val", + _display_path(request.validation_path), + "--optimizer-config", + _display_path(request.optimizer_config_path), + "--output-dir", + "$OUTPUT_DIR", + "--run-id", + request.run_id, + ] + target_paths = list(request.target_prompt_paths.items()) + if set(request.target_prompt_paths) == {"system_prompt"}: + command.extend(["--prompt", _display_path(request.target_prompt_paths["system_prompt"])]) + else: + for name, path in target_paths: + command.extend(["--target-prompt", f"{name}={_display_path(path)}"]) + if request.trace: + command.append("--trace") + if request.mode == "sdk" and request.sdk_call_agent: + command.extend(["--sdk-call-agent", request.sdk_call_agent]) + if request.gate_config_path is not None: + command.extend(["--gate-config", _display_path(request.gate_config_path)]) + if request.update_source: + command.append("--update-source") + return " ".join(_powershell_arg(item) for item in command) + + +def _powershell_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _powershell_arg(value: str) -> str: + placeholder = re.search(r"\$(OUTPUT_DIR|EXTERNAL)", value) + if placeholder is not None: + prefix = value[:placeholder.start()] + suffix = value[placeholder.end():] + + def escape_literal(text: str) -> str: + return text.replace("`", "``").replace("$", "`$").replace('"', '`"') + + return ('"' + escape_literal(prefix) + placeholder.group(0) + escape_literal(suffix) + '"') + if re.fullmatch(r"[A-Za-z0-9_./:-]+", value): + return value + return _powershell_quote(value) + + +def _initial_writeback_journal( + *, + run_id: str, + selected_candidate: str | None, + update_source: bool, + before_hashes: dict[str, str], + candidate_hashes: dict[str, str], + input_hashes: dict[str, str], +) -> dict[str, Any]: + if selected_candidate is None: + state = "rejected" + elif not update_source: + state = "not_requested" + else: + state = "pending" + return { + "run_id": run_id, + "state": state, + "report_phase": "pre_write", + "requested": update_source, + "selected_candidate": selected_candidate, + "before_hashes": dict(before_hashes), + "candidate_hashes": dict(candidate_hashes), + "input_hashes": dict(input_hashes), + "after_hashes": {}, + "error": None, + } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py new file mode 100644 index 00000000..1734c851 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -0,0 +1,1022 @@ +"""Report construction and rendering.""" + +from __future__ import annotations + +import ctypes +import errno +import json +import hashlib +import os +import stat +import sys +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .attribution import summarize_failures +from .artifacts import validate_artifact_component +from .schemas import CandidatePrompt +from .schemas import CaseDelta +from .schemas import CostSummary +from .schemas import EvalResult +from .schemas import GateDecision +from .schemas import OptimizationReport +from .schemas import OptimizationRound +from .schemas import WritebackResult +from .schemas import to_jsonable + +REPRODUCIBILITY_COMMAND = ("python examples/optimization/eval_optimize_loop/run_pipeline.py " + "--train examples/optimization/eval_optimize_loop/data/train.evalset.json " + "--val examples/optimization/eval_optimize_loop/data/val.evalset.json " + "--optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json " + "--prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt " + "--output-dir /tmp/eval-optimize-loop " + "--fake-model --fake-judge --trace") + + +@dataclass(frozen=True) +class RunArtifactPaths: + """Locations written by the audit-first report lifecycle.""" + + output_dir: Path + run_id: str + temp_run_dir: Path + final_run_dir: Path + + @property + def run_dir(self) -> Path: + """Return the immutable authoritative run location.""" + + return self.final_run_dir + + @property + def prewrite_json(self) -> Path: + return self.final_run_dir / "pre_write_report.json" + + @property + def prewrite_markdown(self) -> Path: + return self.final_run_dir / "pre_write_report.md" + + @property + def audit_json(self) -> Path: + return self.final_run_dir / "audit.json" + + @property + def final_json(self) -> Path: + return self.final_run_dir / "optimization_report.json" + + @property + def final_markdown(self) -> Path: + return self.final_run_dir / "optimization_report.md" + + @property + def writeback_json(self) -> Path: + return self.final_run_dir / "writeback.json" + + @property + def writeback_journal(self) -> Path: + return self.final_run_dir / "writeback_journal.json" + + @property + def top_level_json(self) -> Path: + return self.output_dir / "optimization_report.json" + + @property + def top_level_markdown(self) -> Path: + return self.output_dir / "optimization_report.md" + + def temp_path(self, relative_path: str | Path) -> Path: + return self.temp_run_dir / relative_path + + +def compute_case_deltas( + *, + candidate_id: str, + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_train: EvalResult, + candidate_validation: EvalResult, +) -> list[CaseDelta]: + deltas: list[CaseDelta] = [] + for baseline, candidate in ((baseline_train, candidate_train), (baseline_validation, candidate_validation)): + candidate_by_id = candidate.by_case_id() + for baseline_case in baseline.cases: + candidate_case = candidate_by_id[baseline_case.case_id] + delta = round(candidate_case.score - baseline_case.score, 6) + delta_type = _delta_type( + baseline_passed=baseline_case.passed, + candidate_passed=candidate_case.passed, + delta=delta, + ) + deltas.append( + CaseDelta( + candidate_id=candidate_id, + case_id=baseline_case.case_id, + split=baseline_case.split, + baseline_score=baseline_case.score, + candidate_score=candidate_case.score, + delta=delta, + baseline_passed=baseline_case.passed, + candidate_passed=candidate_case.passed, + regression=delta < 0, + delta_type=delta_type, + )) + return deltas + + +def build_report( + *, + run: dict[str, Any], + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_records: list[dict[str, Any]], + per_case_deltas: list[CaseDelta], + gate_decisions: list[GateDecision], + selected_candidate: str | None, + audit: dict[str, Any], + baseline_prompts: dict[str, str] | None = None, + rounds: list[OptimizationRound] | None = None, + cost_summary: CostSummary | None = None, + writeback: WritebackResult | None = None, +) -> OptimizationReport: + all_results: list[EvalResult] = [baseline_train, baseline_validation] + for record in candidate_records: + for result_name in ("train_result", "validation_result"): + result = record.get(result_name) + if isinstance(result, EvalResult): + all_results.append(result) + return OptimizationReport( + schema_version="eval_optimize_loop.v2", + run=run, + baseline={ + "train": baseline_train, + "validation": baseline_validation + }, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidates=candidate_records, + delta={"per_case": per_case_deltas}, + per_case_deltas=per_case_deltas, + failure_attribution_summary=summarize_failures(all_results), + gate_decisions=gate_decisions, + selected_candidate=selected_candidate, + audit=audit, + baseline_prompts=dict(baseline_prompts or {}), + rounds=list(rounds or []), + cost_summary=cost_summary or CostSummary(), + writeback=writeback or WritebackResult(status="not_requested"), + ) + + +def validate_unique_round_ids(rounds: list[OptimizationRound]) -> None: + """Reject round collections that cannot map one-to-one to audit files.""" + + seen: set[int] = set() + for round_record in rounds: + round_id = round_record.round_id + if round_id in seen: + raise ValueError(f"duplicate round_id in optimization report: {round_id}") + seen.add(round_id) + + +def write_reports(report: OptimizationReport, output_dir: str | Path) -> tuple[Path, Path]: + """Compatibility helper for callers that do not perform source writeback.""" + + run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) + paths = reserve_run_artifacts(output_dir, run_id=run_id) + prepare_run_artifacts(report, paths) + finalize_run_artifacts(report, paths) + return paths.top_level_json, paths.top_level_markdown + + +def reserve_run_artifacts( + output_dir: str | Path, + *, + run_id: str, +) -> RunArtifactPaths: + """Exclusively reserve one mutable temp run without touching a final run.""" + + safe_run_id = _safe_artifact_name(str(run_id)) + output_path = Path(output_dir) + runs_root = output_path / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + if runs_root.is_symlink() or _is_reparse_point(runs_root): + raise OSError(f"refusing to reserve run artifacts through unsafe runs directory: {runs_root}") + + temp_run_dir = runs_root / f".{safe_run_id}.tmp" + final_run_dir = runs_root / safe_run_id + for occupied in (final_run_dir, temp_run_dir): + if os.path.lexists(occupied): + raise FileExistsError(f"run ID {safe_run_id!r} is already reserved or published in {runs_root}") + try: + temp_run_dir.mkdir() + except FileExistsError as error: + raise FileExistsError(f"run ID {safe_run_id!r} is already reserved or published in {runs_root}") from error + + # A concurrently published final must win. Leave no misleading reservation + # when our just-created directory is still empty. + if os.path.lexists(final_run_dir): + try: + temp_run_dir.rmdir() + except OSError: + pass + raise FileExistsError(f"run ID {safe_run_id!r} is already reserved or published in {runs_root}") + return RunArtifactPaths( + output_dir=output_path, + run_id=safe_run_id, + temp_run_dir=temp_run_dir, + final_run_dir=final_run_dir, + ) + + +def prepare_run_artifacts( + report: OptimizationReport, + paths_or_output_dir: RunArtifactPaths | str | Path, +) -> RunArtifactPaths: + """Persist a complete pre-write audit before any source prompt commit.""" + + report_run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) + if isinstance(paths_or_output_dir, RunArtifactPaths): + paths = paths_or_output_dir + else: + # Legacy direct callers remain safe: they receive a new exclusive + # reservation and can never reopen an existing temp or final run. + paths = reserve_run_artifacts(paths_or_output_dir, run_id=report_run_id) + validate_unique_round_ids(report.rounds) + _validate_run_artifact_paths(paths, report_run_id=report_run_id) + _require_mutable_temp_run(paths) + + prewrite_json = report_to_json(report) + prewrite_markdown = render_markdown(report) + audit_json = _json_text(report.audit) + journal = dict(report.audit.get("writeback_journal", {})) + if journal.get("state") == "pending": + journal["state"] = "prepared" + journal_json = _json_text(journal) + + _atomic_write_text(paths.temp_path("pre_write_report.json"), prewrite_json) + _atomic_write_text(paths.temp_path("pre_write_report.md"), prewrite_markdown) + _atomic_write_text(paths.temp_path("audit.json"), audit_json) + write_audit_artifacts(report, paths.temp_run_dir) + _write_artifact_manifest(report, paths, expected_journal=journal_json) + # This durable journal is the completion marker for the whole prepare phase, + # so it must be replaced only after every prerequisite artifact exists. + persist_writeback_journal(paths, journal) + return paths + + +def finalize_run_artifacts( + report: OptimizationReport, + paths: RunArtifactPaths, + *, + before_publish: Callable[[], None] | None = None, +) -> None: + """Complete the temp run, publish it once, then refresh convenience copies.""" + + report_run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) + validate_unique_round_ids(report.rounds) + _validate_run_artifact_paths(paths, report_run_id=report_run_id) + _require_mutable_temp_run(paths) + + # Serialize everything before the first final-phase write so non-finite or + # otherwise invalid values cannot create a misleading terminal artifact set. + audit_json = _json_text(report.audit) + final_json = report_to_json(report) + final_markdown = render_markdown(report) + + # The terminal outcome is authoritative within the still-mutable temp run. + persist_writeback_outcome(report, paths) + _atomic_write_text(paths.temp_path("audit.json"), audit_json) + _atomic_write_text(paths.temp_path("optimization_report.json"), final_json) + _atomic_write_text(paths.temp_path("optimization_report.md"), final_markdown) + write_audit_artifacts(report, paths.temp_run_dir) + _write_artifact_manifest(report, paths) + + # This is intentionally the last callback before the one-way directory + # rename. It may downgrade the temp journal and raise, but publication makes + # every final-run byte immutable to this lifecycle. + if before_publish is not None: + try: + before_publish() + except BaseException as callback_error: + try: + _write_artifact_manifest(report, paths) + except BaseException as manifest_error: + add_note = getattr(callback_error, "add_note", None) + if add_note is not None: + add_note("failed to refresh temp artifact manifest after before_publish failure: " + f"{manifest_error}") + raise + _durable_publish_directory(paths) + + # Convenience copies are non-authoritative and are refreshed only from the + # already-published immutable run. Failure here leaves that final intact. + _atomic_write_bytes(paths.top_level_json, paths.final_json.read_bytes()) + _atomic_write_bytes(paths.top_level_markdown, paths.final_markdown.read_bytes()) + + +def persist_writeback_journal(paths: RunArtifactPaths, journal: dict[str, Any]) -> None: + """Atomically persist the authoritative writeback state machine record.""" + + _require_mutable_temp_run(paths) + _atomic_write_text(paths.temp_path("writeback_journal.json"), _json_text(journal)) + + +def persist_writeback_outcome(report: OptimizationReport, paths: RunArtifactPaths) -> None: + """Persist a terminal writeback outcome before any nonessential artifact.""" + + persist_writeback_journal(paths, dict(report.audit.get("writeback_journal", {}))) + _atomic_write_text(paths.temp_path("writeback.json"), _json_text(report.writeback)) + + +def _validate_run_artifact_paths(paths: RunArtifactPaths, *, report_run_id: str) -> None: + runs_root = paths.output_dir / "runs" + if paths.run_id != report_run_id: + raise ValueError("run artifact paths do not match report run_id") + if paths.temp_run_dir != runs_root / f".{report_run_id}.tmp": + raise ValueError("run artifact temp path does not match report run_id") + if paths.final_run_dir != runs_root / report_run_id: + raise ValueError("run artifact final path does not match report run_id") + + +def _require_mutable_temp_run(paths: RunArtifactPaths) -> None: + if os.path.lexists(paths.final_run_dir): + raise FileExistsError(f"run ID {paths.run_id!r} is already published") + try: + metadata = os.lstat(paths.temp_run_dir) + except FileNotFoundError as error: + raise FileNotFoundError(f"reserved temp directory for run ID {paths.run_id!r} is unavailable") from error + if paths.temp_run_dir.is_symlink() or _is_reparse_point(paths.temp_run_dir): + raise OSError(f"refusing unsafe reserved temp directory: {paths.temp_run_dir}") + if not stat.S_ISDIR(metadata.st_mode): + raise OSError(f"reserved temp path is not a directory: {paths.temp_run_dir}") + + +def report_to_json(report: OptimizationReport) -> str: + validate_unique_round_ids(report.rounds) + return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True, allow_nan=False) + "\n" + + +def _json_text(value: Any) -> str: + return (json.dumps( + to_jsonable(value), + indent=2, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + "\n") + + +def _atomic_write_text(path: Path, content: str) -> None: + _atomic_write_bytes(path, content.encode("utf-8")) + + +def _atomic_write_bytes(path: Path, content: bytes) -> None: + """Durably replace a critical artifact without exposing a partial file.""" + + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + stream = os.fdopen(fd, "wb") + fd = -1 + with stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + _durable_replace(temp_path, path) + finally: + active_exception = sys.exc_info()[0] is not None + cleanup_error: OSError | None = None + if fd >= 0: + try: + os.close(fd) + except OSError as error: + cleanup_error = error + try: + temp_path.unlink() + except FileNotFoundError: + pass + except OSError as error: + if cleanup_error is None: + cleanup_error = error + if cleanup_error is not None and not active_exception: + raise cleanup_error + + +def _fsync_directory(directory: Path) -> None: + """Synchronize a POSIX directory entry after atomic replacement.""" + + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + fd = os.open(directory, flags) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _durable_replace(source: Path, target: Path) -> None: + """Atomically replace a file and wait for rename metadata to reach storage.""" + + if os.name == "nt": + if _is_reparse_point(target): + raise OSError(f"refusing to replace reparse-point target: {target}") + move_file_ex = ctypes.WinDLL("kernel32", use_last_error=True).MoveFileExW + move_file_ex.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32] + move_file_ex.restype = ctypes.c_int + movefile_replace_existing = 0x00000001 + movefile_write_through = 0x00000008 + succeeded = move_file_ex( + os.path.abspath(source), + os.path.abspath(target), + movefile_replace_existing | movefile_write_through, + ) + if not succeeded: + raise ctypes.WinError(ctypes.get_last_error()) + return + os.replace(source, target) + _fsync_directory(target.parent) + + +def _durable_publish_directory(paths: RunArtifactPaths) -> None: + """Atomically move the reserved temp run to a never-replaced final path.""" + + _require_mutable_temp_run(paths) + source = paths.temp_run_dir + target = paths.final_run_dir + if target.is_symlink() or _is_reparse_point(target): + raise OSError(f"refusing to publish over unsafe/reparse target: {target}") + if os.path.lexists(target): + raise FileExistsError(f"run ID {paths.run_id!r} is already published") + + if os.name == "nt": + move_file_ex = ctypes.WinDLL("kernel32", use_last_error=True).MoveFileExW + move_file_ex.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32] + move_file_ex.restype = ctypes.c_int + movefile_write_through = 0x00000008 + succeeded = move_file_ex( + os.path.abspath(source), + os.path.abspath(target), + movefile_write_through, + ) + if not succeeded: + error_code = ctypes.get_last_error() + if os.path.lexists(target): + raise FileExistsError( + error_code, + f"run ID {paths.run_id!r} is already published", + str(target), + ) + raise ctypes.WinError(error_code) + return + + _posix_rename_noreplace(source, target) + _fsync_directory(target.parent) + + +def _posix_rename_noreplace(source: Path, target: Path) -> None: + """Atomically rename a Linux directory without replacing any target.""" + + try: + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = libc.renameat2 + except (AttributeError, OSError) as error: + raise OSError( + errno.ENOTSUP, + "atomic POSIX no-replace directory rename is unavailable", + str(target), + ) from error + + at_fdcwd = -100 + rename_noreplace = 0x1 + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + at_fdcwd, + os.fsencode(source), + at_fdcwd, + os.fsencode(target), + rename_noreplace, + ) + if result == 0: + return + + error_number = ctypes.get_errno() or errno.EIO + if error_number in {errno.EEXIST, errno.ENOTEMPTY}: + raise FileExistsError( + error_number, + os.strerror(error_number), + str(target), + ) + raise OSError(error_number, os.strerror(error_number), str(target)) + + +def _is_reparse_point(path: Path) -> bool: + try: + metadata = os.lstat(path) + except FileNotFoundError: + return False + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x00000400) + return bool(getattr(metadata, "st_file_attributes", 0) & reparse_flag) + + +def render_markdown(report: OptimizationReport) -> str: + validate_unique_round_ids(report.rounds) + decision_by_id = {decision.candidate_id: decision for decision in report.gate_decisions} + lines = [ + "# Evaluation + Optimization Report", + "", + "## Final Decision", + "", + ] + if report.selected_candidate: + lines.append(f"Selected candidate: `{report.selected_candidate}`.") + else: + lines.append("No candidate was accepted.") + + lines.extend([ + "", + "Update source prompt: " + ("yes" if report.run.get("update_source") else "no (default)"), + "", + ]) + availability = report.audit.get("sdk_result_availability", {}) + if report.audit.get("candidate_evaluation_failures"): + lines.extend([ + "Fake and SDK modes require complete AgentEvaluator-compatible train and validation " + "reevaluation before gating a candidate; evaluation errors are recorded as explicit " + "rejections and optimizer aggregates are never used as gate evidence.", + "", + ]) + else: + lines.extend([ + "Fake and SDK modes perform complete AgentEvaluator-compatible reevaluation for baseline " + "and every candidate on both train and validation; optimizer aggregates are never used " + "as gate evidence.", + "", + ]) + if report.run.get("mode") == "sdk": + lines.extend([ + "SDK availability: " + f"aggregate_validation_result={availability.get('aggregate_validation_result')}, " + f"full_train_eval_result={availability.get('full_train_eval_result')}, " + f"full_per_case_validation_delta={availability.get('full_per_case_validation_delta')}.", + "", + ]) + lines.extend([ + "", + "## Gate Reasons", + "", + ]) + for decision in report.gate_decisions: + verdict = "accepted" if decision.accepted else "rejected" + lines.append(f"### {decision.candidate_id} ({verdict})") + for reason in decision.reasons: + lines.append(f"- {reason}") + lines.append("") + + lines.extend([ + "## Baseline vs Candidate Scores", + "", + "| prompt | train score | validation score | gate |", + "| --- | ---: | ---: | --- |", + f"| baseline | {report.baseline_train.score:.3f} | {report.baseline_validation.score:.3f} | n/a |", + ]) + for record in report.candidates: + candidate: CandidatePrompt = record["candidate"] + gate = decision_by_id[candidate.candidate_id] + verdict = "accept" if gate.accepted else "reject" + train_result = record.get("train_result") + validation_result = record.get("validation_result") + train_score = f"{train_result.score:.3f}" if isinstance(train_result, EvalResult) else "n/a" + validation_score = (f"{validation_result.score:.3f}" if isinstance(validation_result, EvalResult) else "n/a") + lines.append(f"| {candidate.candidate_id} | {train_score} | " + f"{validation_score} | {verdict} |") + + lines.extend([ + "", + "## Per-Case Delta", + "", + "| candidate | split | case | baseline | candidate | delta | passed -> passed | delta type |", + "| --- | --- | --- | ---: | ---: | ---: | --- | --- |", + ]) + for delta in report.per_case_deltas: + lines.append(f"| {delta.candidate_id} | {delta.split} | {delta.case_id} | " + f"{delta.baseline_score:.3f} | {delta.candidate_score:.3f} | " + f"{delta.delta:+.3f} | {delta.baseline_passed} -> {delta.candidate_passed} | " + f"{delta.delta_type} |") + + summary = report.failure_attribution_summary + lines.extend([ + "", + "## Failure Attribution Summary", + "", + f"Total failed case evaluations: {summary['total_failed_cases']}", + "", + "| category | count |", + "| --- | ---: |", + ]) + by_category = summary.get("by_category", {}) + if by_category: + for category, count in by_category.items(): + lines.append(f"| {category} | {count} |") + else: + lines.append("| none | 0 |") + if summary.get("attribution_accuracy") is not None: + lines.append("") + lines.append(f"Attribution accuracy: {summary['attribution_accuracy']:.3f}") + + cost_audit = report.audit.get("cost", {}) + lines.extend(["", "## Cost And Audit", ""]) + if cost_audit.get("complete"): + lines.append(f"Total cost: {cost_audit.get('total', 0):.3f}") + else: + reported_optimizer_cost = cost_audit.get("reported_optimizer_cost") + if reported_optimizer_cost is not None: + lines.append("Reported optimizer cost (incomplete; not total run cost): " + f"{reported_optimizer_cost:.3f}") + lines.append(f"Known evaluator cost: {cost_audit.get('evaluator', 0):.3f}") + lines.append("Known run cost (incomplete; not total run cost): " + f"{cost_audit.get('known_run_cost', 0):.3f}") + lines.append("Complete run cost: unavailable") + lines.extend([ + f"Config hash: `{report.audit.get('config_hash', '')}`", + f"Run id: `{report.run.get('run_id', '')}`", + ]) + + lines.extend([ + "", + "## Prompt Diff", + "", + ]) + for record in report.candidates: + candidate = record["candidate"] + lines.extend([ + f"### {candidate.candidate_id}", + "", + "```diff", + candidate.prompt_diff, + "```", + "", + ]) + + lines.extend([ + "## Reproducibility", + "", + f"```{report.run.get('reproducibility_shell') or 'bash'}", + report.run.get("reproducibility_command") or report.audit.get("reproducibility_command") + or REPRODUCIBILITY_COMMAND, + "```", + "", + ]) + return "\n".join(lines) + + +def write_audit_artifacts(report: OptimizationReport, run_dir: Path) -> None: + """Write the complete structured audit set inside a mutable run root.""" + + validate_unique_round_ids(report.rounds) + run_dir = Path(run_dir) + _require_safe_audit_root(run_dir) + + for component in ( + "case_results", + "baseline_prompts", + "candidate_prompts", + "prompt_diffs", + "rounds", + ): + _ensure_safe_audit_directory(run_dir, component) + for index, record in enumerate(report.candidates, start=1): + candidate: CandidatePrompt = record["candidate"] + _ensure_safe_audit_directory( + run_dir, + "candidate_prompts", + _candidate_artifact_name(index, candidate.candidate_id), + ) + + # Never copy the raw optimizer file: it may contain API keys. The original + # byte hash remains in input_hashes while this human-readable snapshot is redacted. + _write_safe_audit_text( + run_dir, + "config.snapshot.json", + _json_text(report.audit.get("config_snapshot", {})), + ) + _write_safe_audit_text( + run_dir, + "input_hashes.json", + _json_text(report.audit.get("input_hashes", {})), + ) + + for field_name, prompt_text in report.baseline_prompts.items(): + field_artifact = _safe_artifact_name(str(field_name)) + _write_safe_audit_text( + run_dir, + Path("baseline_prompts") / f"{field_artifact}.txt", + prompt_text, + ) + + _write_safe_audit_text( + run_dir, + Path("case_results") / "baseline_train.json", + _json_text(report.baseline_train), + ) + _write_safe_audit_text( + run_dir, + Path("case_results") / "baseline_validation.json", + _json_text(report.baseline_validation), + ) + _write_safe_audit_text(run_dir, "rounds.json", _json_text(report.rounds)) + for round_record in report.rounds: + round_artifact = _safe_artifact_name(str(round_record.round_id)) + _write_safe_audit_text( + run_dir, + Path("rounds") / f"{round_artifact}.json", + _json_text(round_record), + ) + _write_safe_audit_text( + run_dir, + "per_case_deltas.json", + _json_text(report.per_case_deltas), + ) + _write_safe_audit_text( + run_dir, + "gate_decisions.json", + _json_text(report.gate_decisions), + ) + _write_safe_audit_text( + run_dir, + "evaluation_failures.json", + _json_text(report.audit.get("candidate_evaluation_failures", {})), + ) + _write_safe_audit_text(run_dir, "writeback.json", _json_text(report.writeback)) + + for index, record in enumerate(report.candidates, start=1): + candidate: CandidatePrompt = record["candidate"] + candidate_name = _candidate_artifact_name(index, candidate.candidate_id) + prompt_bundle = record.get("prompt_bundle") + if prompt_bundle is None: + prompt_bundle = candidate.bundle() + for field_name, prompt_text in prompt_bundle.items(): + field_artifact = _safe_artifact_name(str(field_name)) + _write_safe_audit_text( + run_dir, + Path("candidate_prompts") / candidate_name / f"{field_artifact}.txt", + prompt_text, + ) + _write_safe_audit_text( + run_dir, + Path("prompt_diffs") / f"{candidate_name}.diff", + candidate.prompt_diff, + ) + for split_name in ("train_result", "validation_result"): + split_result = record.get(split_name) + if not isinstance(split_result, EvalResult): + continue + split_artifact = _safe_artifact_name(str(split_result.split)) + _write_safe_audit_text( + run_dir, + Path("case_results") / f"{candidate_name}_{split_artifact}.json", + _json_text(split_result), + ) + + +def _require_safe_audit_root(run_root: Path) -> None: + run_root = Path(run_root) + try: + metadata = os.lstat(run_root) + except FileNotFoundError as error: + raise OSError(f"audit run root is unavailable: {run_root}") from error + if stat.S_ISLNK(metadata.st_mode) or _is_reparse_point(run_root): + raise OSError(f"audit run root is unsafe/reparse: {run_root}") + if not stat.S_ISDIR(metadata.st_mode): + raise OSError(f"audit run root is not a directory: {run_root}") + + +def _write_safe_audit_text( + run_root: Path, + relative_path: str | Path, + content: str, +) -> None: + """Recheck an audit file's root and parent immediately before writing.""" + + run_root = Path(run_root) + relative = Path(relative_path) + if (relative.is_absolute() or not relative.parts + or any(component in {"", ".", ".."} for component in relative.parts)): + raise OSError(f"unsafe audit artifact path: {relative_path!r}") + if len(relative.parts) == 1: + _require_safe_audit_root(run_root) + parent = run_root + else: + parent = _ensure_safe_audit_directory(run_root, *relative.parts[:-1]) + target = parent / relative.parts[-1] + try: + target.relative_to(run_root) + except ValueError as error: + raise OSError(f"audit artifact escapes reserved run root: {target}") from error + _atomic_write_text(target, content) + + +def _ensure_safe_audit_directory(run_root: Path, *components: str) -> Path: + """Return a checked audit directory rooted lexically beneath ``run_root``. + + Every existing component is inspected without following links. Missing + components are created one level at a time and immediately re-inspected. + Callers use the returned directory directly for the following writes; this + eliminates deterministic escapes left by a backend, but does not claim a + portable cross-process no-follow guarantee after the final inspection. + """ + + run_root = Path(run_root) + _require_safe_audit_root(run_root) + if not components: + raise ValueError("audit directory components must not be empty") + for component in components: + component_path = Path(component) + if (not component or component_path.is_absolute() or len(component_path.parts) != 1 + or component_path.name != component or component in {".", ".."}): + raise OSError(f"unsafe audit directory component: {component!r}") + + directory = run_root.joinpath(*components) + try: + directory.relative_to(run_root) + except ValueError as error: + raise OSError(f"audit directory escapes reserved run root: {directory}") from error + + current = run_root + for component in components: + current = current / component + try: + metadata = os.lstat(current) + except FileNotFoundError: + try: + os.mkdir(current) + except FileExistsError: + # A racing creator must pass the same no-follow inspection. + pass + metadata = os.lstat(current) + if stat.S_ISLNK(metadata.st_mode) or _is_reparse_point(current): + raise OSError(f"refusing unsafe/reparse audit directory: {current}") + if not stat.S_ISDIR(metadata.st_mode): + raise OSError(f"audit path component is not a directory: {current}") + return directory + + +def _write_artifact_manifest( + report: OptimizationReport, + paths: RunArtifactPaths, + *, + expected_journal: str | None = None, +) -> None: + _require_mutable_temp_run(paths) + artifacts = _manifest_artifact_layout(report) + expected_files = ({ + "writeback_journal.json": expected_journal.encode("utf-8") + } if expected_journal is not None else {}) + files = _manifest_file_records(paths.temp_run_dir, expected_files=expected_files) + declared_paths = _declared_manifest_paths(artifacts) + if expected_journal is not None: + declared_paths -= {"optimization_report.json", "optimization_report.md"} + available_paths = {item["path"] for item in files} + missing = declared_paths - available_paths + if missing: + raise OSError(f"audit artifact manifest is incomplete: missing {sorted(missing)}") + manifest = { + "schema_version": "eval_optimize_loop.artifacts.v1", + "run_id": paths.run_id, + "artifacts": artifacts, + "files": files, + } + _write_safe_audit_text( + paths.temp_run_dir, + "artifact_manifest.json", + _json_text(manifest), + ) + + +def _manifest_artifact_layout(report: OptimizationReport) -> dict[str, Any]: + validate_unique_round_ids(report.rounds) + candidates: list[dict[str, Any]] = [] + for index, record in enumerate(report.candidates, start=1): + candidate: CandidatePrompt = record["candidate"] + artifact_id = _candidate_artifact_name(index, candidate.candidate_id) + prompt_bundle = record.get("prompt_bundle") or candidate.bundle() + prompt_paths = { + field_name: + (Path("candidate_prompts") / artifact_id / f"{_safe_artifact_name(str(field_name))}.txt").as_posix() + for field_name in prompt_bundle + } + case_results: dict[str, str] = {} + for result_name in ("train_result", "validation_result"): + result = record.get(result_name) + if not isinstance(result, EvalResult): + continue + split = _safe_artifact_name(str(result.split)) + case_results[str(result.split)] = (Path("case_results") / f"{artifact_id}_{split}.json").as_posix() + candidates.append({ + "candidate_id": candidate.candidate_id, + "artifact_id": artifact_id, + "prompt_bundle": prompt_paths, + "diff": (Path("prompt_diffs") / f"{artifact_id}.diff").as_posix(), + "case_results": case_results, + "evaluation_failure": "evaluation_error" in record, + }) + + return { + "reports": { + "prewrite_json": "pre_write_report.json", + "prewrite_markdown": "pre_write_report.md", + "final_json": "optimization_report.json", + "final_markdown": "optimization_report.md", + }, + "baseline_case_results": { + "train": "case_results/baseline_train.json", + "validation": "case_results/baseline_validation.json", + }, + "baseline_prompts": { + field_name: (Path("baseline_prompts") / f"{_safe_artifact_name(str(field_name))}.txt").as_posix() + for field_name in report.baseline_prompts + }, + "rounds": [(Path("rounds") / f"{_safe_artifact_name(str(round_record.round_id))}.json").as_posix() + for round_record in report.rounds], + "audit_records": { + "audit": "audit.json", + "config_snapshot": "config.snapshot.json", + "input_hashes": "input_hashes.json", + "rounds": "rounds.json", + "per_case_deltas": "per_case_deltas.json", + "gate_decisions": "gate_decisions.json", + "writeback": "writeback.json", + "writeback_journal": "writeback_journal.json", + "evaluation_failures": "evaluation_failures.json", + }, + "candidates": + candidates, + } + + +def _declared_manifest_paths(artifacts: dict[str, Any]) -> set[str]: + declared: set[str] = set() + for group_name in ("reports", "baseline_case_results", "baseline_prompts", "audit_records"): + declared.update(artifacts[group_name].values()) + declared.update(artifacts["rounds"]) + for candidate in artifacts["candidates"]: + declared.add(candidate["diff"]) + declared.update(candidate["prompt_bundle"].values()) + declared.update(candidate["case_results"].values()) + return declared + + +def _manifest_file_records( + run_dir: Path, + *, + expected_files: dict[str, bytes], +) -> list[dict[str, Any]]: + content_by_path: dict[str, bytes] = dict(expected_files) + for artifact in run_dir.rglob("*"): + if artifact.is_symlink() or _is_reparse_point(artifact): + raise OSError(f"refusing unsafe/reparse audit artifact: {artifact}") + if not artifact.is_file(): + continue + relative_path = artifact.relative_to(run_dir).as_posix() + if relative_path == "artifact_manifest.json": + continue + content_by_path[relative_path] = artifact.read_bytes() + return [{ + "path": relative_path, + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + } for relative_path, content in sorted(content_by_path.items())] + + +def _candidate_artifact_name(index: int, candidate_id: str) -> str: + digest = hashlib.sha256(candidate_id.encode("utf-8")).hexdigest()[:12] + return f"{index:03d}-{digest}" + + +def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float) -> str: + if not baseline_passed and candidate_passed: + return "new_pass" + if baseline_passed and not candidate_passed: + return "new_fail" + if delta > 0: + return "score_up" + if delta < 0: + return "score_down" + return "unchanged" + + +def _safe_artifact_name(name: str) -> str: + return validate_artifact_component(name, context="audit artifact name") diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py new file mode 100644 index 00000000..58afb147 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -0,0 +1,229 @@ +"""Dataclass schemas used by the example optimization loop.""" + +from __future__ import annotations + +from dataclasses import asdict +from dataclasses import dataclass +from dataclasses import field +from dataclasses import is_dataclass +from typing import Any +from typing import Literal + + +@dataclass(frozen=True) +class EvalCase: + """One deterministic evaluation case.""" + + case_id: str + split: str + input: str + expectation: dict[str, Any] + tags: list[str] = field(default_factory=list) + protected: bool = False + simulated_outputs: dict[str, str] = field(default_factory=dict) + expected_failure_category: str | None = None + + @classmethod + def from_dict(cls, payload: dict[str, Any], split: str) -> "EvalCase": + case_id = payload.get("case_id") or payload.get("id") + if not case_id: + raise ValueError(f"eval case is missing id/case_id: {payload!r}") + if "split" in payload and str(payload["split"]) != str(split): + raise ValueError( + f"eval case {case_id!r} split mismatch: payload has {payload['split']!r}, expected {split!r}") + expectation = payload.get("expectation") + if not isinstance(expectation, dict): + raise ValueError(f"eval case {case_id!r} is missing expectation object") + return cls( + case_id=str(case_id), + split=str(split), + input=str(payload.get("input") or payload.get("user_input") or ""), + expectation=dict(expectation), + tags=list(payload.get("tags") or []), + protected=bool(payload.get("protected", False)), + simulated_outputs=dict(payload.get("simulated_outputs") or expectation.get("simulated_outputs") or {}), + expected_failure_category=payload.get("expected_failure_category") + or expectation.get("expected_failure_category"), + ) + + +@dataclass(frozen=True) +class CaseResult: + """Evaluation result for one case under one prompt.""" + + case_id: str + split: str + score: float + passed: bool + output: str + metrics: dict[str, float] = field(default_factory=dict, kw_only=True) + trace: dict[str, Any] = field(default_factory=dict) + trace_available: bool = field(default=False, kw_only=True) + failure_category: str | None = None + failure_reason: str | None = None + evidence: str | None = None + cost: float = 0.0 + hard_failed: bool = False + expected_failure_category: str | None = None + + +@dataclass(frozen=True) +class EvalResult: + """Aggregate result for one prompt on one split.""" + + prompt_id: str + split: str + score: float + passed: bool + cost: float + cases: list[CaseResult] + + def by_case_id(self) -> dict[str, CaseResult]: + return {case.case_id: case for case in self.cases} + + +@dataclass(frozen=True) +class CandidatePrompt: + """A proposed prompt candidate.""" + + candidate_id: str + prompt: str + rationale: str + prompt_diff: str + prompt_fields: dict[str, str] = field(default_factory=dict) + + def bundle(self) -> dict[str, str]: + """Return this candidate's complete prompt bundle.""" + + if self.prompt_fields: + return dict(self.prompt_fields) + return {"system_prompt": self.prompt} + + +@dataclass(frozen=True) +class CostSummary: + """Cost attribution for an optimization run.""" + + optimizer: float = 0.0 + evaluator: float = 0.0 + agent: float = 0.0 + total: float = 0.0 + complete: bool = True + reported_optimizer_cost: float | None = None + + +@dataclass(frozen=True) +class OptimizationRound: + """One auditable optimizer round.""" + + round_id: int + candidate_id: str + prompts: dict[str, str] + rationale: str + metrics: dict[str, float] + cost: CostSummary + duration_seconds: float + + +WritebackStatus = Literal[ + "rejected", + "not_requested", + "applied", + "rolled_back", + "rollback_failed", +] + + +@dataclass(frozen=True) +class WritebackResult: + """Outcome of an optional source prompt writeback.""" + + status: WritebackStatus + before_hashes: dict[str, str] = field(default_factory=dict) + after_hashes: dict[str, str] = field(default_factory=dict) + error: str | None = None + + +@dataclass(frozen=True) +class OptimizationResult: + """Backend-neutral optimization output.""" + + candidates: list[CandidatePrompt] + rounds: list[OptimizationRound] + cost: CostSummary + raw_summary: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CaseDelta: + """Per-case score delta from baseline to candidate.""" + + candidate_id: str + case_id: str + split: str + baseline_score: float + candidate_score: float + delta: float + baseline_passed: bool + candidate_passed: bool + regression: bool + delta_type: str + + +@dataclass(frozen=True) +class GateDecision: + """Configurable acceptance gate result for one candidate.""" + + candidate_id: str + accepted: bool + reasons: list[str] + train_score_delta: float | None + validation_score_delta: float | None + new_hard_failures: list[str] + protected_regressions: list[str] + validation_new_failures: list[str] + excessive_score_drops: list[str] + overfit_detected: bool | None + candidate_cost: float + cumulative_cost: float + total_run_cost: float + cost: float + gate_status: str = "applied" + gate_not_applied_reason: str | None = None + not_applied_checks: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class OptimizationReport: + """Complete persisted audit report for the loop.""" + + schema_version: str + run: dict[str, Any] + baseline: dict[str, EvalResult] + baseline_train: EvalResult + baseline_validation: EvalResult + candidates: list[dict[str, Any]] + delta: dict[str, Any] + per_case_deltas: list[CaseDelta] + failure_attribution_summary: dict[str, Any] + gate_decisions: list[GateDecision] + selected_candidate: str | None + audit: dict[str, Any] + rounds: list[OptimizationRound] = field(default_factory=list) + cost_summary: CostSummary = field(default_factory=CostSummary) + writeback: WritebackResult = field(default_factory=lambda: WritebackResult(status="not_requested")) + baseline_prompts: dict[str, str] = field(default_factory=dict) + + +def to_jsonable(value: Any) -> Any: + """Convert dataclasses and nested containers into JSON-serializable data.""" + + if is_dataclass(value): + return {key: to_jsonable(item) for key, item in asdict(value).items()} + if isinstance(value, dict): + return {str(key): to_jsonable(item) for key, item in value.items()} + if isinstance(value, list): + return [to_jsonable(item) for item in value] + if isinstance(value, tuple): + return [to_jsonable(item) for item in value] + return value diff --git a/examples/optimization/eval_optimize_loop/eval_loop/trace.py b/examples/optimization/eval_optimize_loop/eval_loop/trace.py new file mode 100644 index 00000000..917416d5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/trace.py @@ -0,0 +1,18 @@ +"""Trace helpers for fake-mode runs.""" + +from __future__ import annotations + +from typing import Any + + +def make_trace(enabled: bool, *, prompt_id: str, case_id: str, model_trace: dict[str, Any], + judge_trace: dict[str, Any]) -> dict[str, Any]: + if not enabled: + return {} + return { + "trace_mode": "fake", + "prompt_id": prompt_id, + "case_id": case_id, + "model": model_trace, + "judge": judge_trace, + } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/writeback.py b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py new file mode 100644 index 00000000..f3a73468 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/writeback.py @@ -0,0 +1,289 @@ +"""Best-effort prompt CAS with per-file atomic replacement. + +Hash checks narrow concurrency windows but cannot make a multi-file operation +transactional. A non-cooperating writer can still race after the last check. +""" + +from __future__ import annotations + +import hashlib +import os +import sys +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path + +from .artifacts import validate_distinct_file_paths +from .schemas import WritebackResult + + +class ConcurrentPromptUpdateError(RuntimeError): + """Raised when source prompts no longer match their captured snapshot.""" + + +class PromptRestorationError(RuntimeError): + """Raised when temporarily installed prompts cannot be restored safely.""" + + +@dataclass(frozen=True) +class PromptFileSnapshot: + """Byte-for-byte snapshot of one source prompt file.""" + + name: str + path: Path + content: bytes + sha256: str + + +@dataclass(frozen=True) +class PromptSnapshot: + """Ordered bundle of source prompt snapshots keyed by prompt name.""" + + files: dict[str, PromptFileSnapshot] + + def hashes(self) -> dict[str, str]: + """Return a fresh name-to-hash mapping for this snapshot.""" + + return {name: prompt_file.sha256 for name, prompt_file in self.files.items()} + + +def _hash_bytes(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def snapshot_prompt_files(paths: dict[str, str | Path]) -> PromptSnapshot: + """Read source prompt files and capture their exact bytes and hashes.""" + + validate_distinct_file_paths(paths, context="prompt snapshot files") + files: dict[str, PromptFileSnapshot] = {} + for name, raw_path in paths.items(): + path = Path(raw_path) + content = path.read_bytes() + files[name] = PromptFileSnapshot( + name=name, + path=path, + content=content, + sha256=_hash_bytes(content), + ) + return PromptSnapshot(files=files) + + +def _current_hashes(snapshot: PromptSnapshot) -> dict[str, str]: + return {name: _hash_bytes(prompt_file.path.read_bytes()) for name, prompt_file in snapshot.files.items()} + + +def _atomic_replace_bytes( + path: Path, + content: bytes, + *, + expected_sha256: str | None = None, +) -> None: + """Replace one file atomically if its last observed hash is still expected.""" + + fd, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + temp_file = os.fdopen(fd, "wb") + fd = -1 + with temp_file: + temp_file.write(content) + temp_file.flush() + os.fsync(temp_file.fileno()) + if expected_sha256 is not None: + current_sha256 = _hash_bytes(path.read_bytes()) + if current_sha256 != expected_sha256: + raise ConcurrentPromptUpdateError(f"source prompt file changed before replace: {path}") + os.replace(temp_path, path) + finally: + active_exception = sys.exc_info()[0] is not None + cleanup_error: OSError | None = None + if fd >= 0: + try: + os.close(fd) + except OSError as error: + cleanup_error = error + try: + temp_path.unlink() + except FileNotFoundError: + pass + except OSError as error: + if cleanup_error is None: + cleanup_error = error + if cleanup_error is not None and not active_exception: + raise cleanup_error + + +def _restore_snapshot( + snapshot: PromptSnapshot, + written_hashes: dict[str, str], +) -> list[str]: + """Conditionally restore only files this operation successfully replaced.""" + + failures: list[str] = [] + for name, candidate_hash in written_hashes.items(): + prompt_file = snapshot.files[name] + try: + current_hash = _hash_bytes(prompt_file.path.read_bytes()) + except OSError as error: + failures.append(f"{name}: restore precondition failed: {error}") + continue + if current_hash != candidate_hash: + failures.append(f"{name}: restore conflict; current hash changed from candidate") + continue + try: + _atomic_replace_bytes( + prompt_file.path, + prompt_file.content, + expected_sha256=candidate_hash, + ) + except (OSError, ConcurrentPromptUpdateError) as error: + failures.append(f"{name}: restore failed: {error}") + return failures + + +def _encode_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> dict[str, bytes]: + missing = [name for name in snapshot.files if name not in prompts] + if missing: + raise ValueError(f"missing prompts for snapshot files: {', '.join(missing)}") + return {name: prompts[name].encode("utf-8") for name in snapshot.files} + + +def _restoration_error(snapshot: PromptSnapshot, failures: list[str]) -> str | None: + details: list[str] = [] + if failures: + details.append(f"restore failures: {'; '.join(failures)}") + + try: + current_hashes = _current_hashes(snapshot) + except OSError as error: + details.append(f"restore verification failed: {error}") + else: + expected_hashes = snapshot.hashes() + mismatched = [name for name, expected_hash in expected_hashes.items() if current_hashes[name] != expected_hash] + if mismatched: + details.append(f"restored hashes differ for: {', '.join(mismatched)}") + + if not details: + return None + return "; ".join(details) + + +@contextmanager +def temporary_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> Iterator[None]: + """Install candidates temporarily using best-effort CAS and conditional restore.""" + + encoded_prompts = _encode_prompt_bundle(snapshot, prompts) + expected_hashes = snapshot.hashes() + current_hashes = _current_hashes(snapshot) + if current_hashes != expected_hashes: + changed = [name for name, expected_hash in expected_hashes.items() if current_hashes[name] != expected_hash] + raise ConcurrentPromptUpdateError(f"source prompt files changed since snapshot: {', '.join(changed)}") + + candidate_hashes = {name: _hash_bytes(content) for name, content in encoded_prompts.items()} + written_hashes: dict[str, str] = {} + try: + for name, prompt_file in snapshot.files.items(): + _atomic_replace_bytes( + prompt_file.path, + encoded_prompts[name], + expected_sha256=expected_hashes[name], + ) + written_hashes[name] = candidate_hashes[name] + installed_hashes = _current_hashes(snapshot) + candidate_mismatches = [ + name for name, candidate_hash in candidate_hashes.items() if installed_hashes[name] != candidate_hash + ] + if candidate_mismatches: + raise ConcurrentPromptUpdateError("candidate prompt files changed before temporary evaluation: " + f"{', '.join(candidate_mismatches)}") + yield + except BaseException as primary_error: + failures = _restore_snapshot(snapshot, written_hashes) + restoration_error = _restoration_error(snapshot, failures) + if restoration_error is not None: + diagnostic = f"failed to restore prompt snapshot: {restoration_error}" + restore_error = PromptRestorationError(diagnostic) + add_note = getattr(primary_error, "add_note", None) + if add_note is not None: + add_note(diagnostic) + raise primary_error from restore_error + raise + else: + failures = _restore_snapshot(snapshot, written_hashes) + restoration_error = _restoration_error(snapshot, failures) + if restoration_error is not None: + raise PromptRestorationError(f"failed to restore prompt snapshot: {restoration_error}") + + +def commit_prompt_bundle( + snapshot: PromptSnapshot, + prompts: dict[str, str], +) -> WritebackResult: + """Apply a bundle with best-effort CAS and conditional compensating rollback.""" + + before_hashes = _current_hashes(snapshot) + expected_hashes = snapshot.hashes() + if before_hashes != expected_hashes: + changed = [name for name, expected_hash in expected_hashes.items() if before_hashes[name] != expected_hash] + raise ConcurrentPromptUpdateError(f"source prompt files changed since snapshot: {', '.join(changed)}") + + encoded_prompts = _encode_prompt_bundle(snapshot, prompts) + candidate_hashes = {name: _hash_bytes(content) for name, content in encoded_prompts.items()} + written_hashes: dict[str, str] = {} + try: + for name, prompt_file in snapshot.files.items(): + _atomic_replace_bytes( + prompt_file.path, + encoded_prompts[name], + expected_sha256=expected_hashes[name], + ) + written_hashes[name] = candidate_hashes[name] + applied_hashes = _current_hashes(snapshot) + candidate_mismatches = [ + name for name, candidate_hash in candidate_hashes.items() if applied_hashes[name] != candidate_hash + ] + if candidate_mismatches: + raise ConcurrentPromptUpdateError("candidate prompt files changed before final verification: " + f"{', '.join(candidate_mismatches)}") + except (OSError, ConcurrentPromptUpdateError) as error: + if isinstance(error, ConcurrentPromptUpdateError) and not written_hashes: + raise + + rollback_failures = _restore_snapshot(snapshot, written_hashes) + try: + after_hashes = _current_hashes(snapshot) + except OSError as hash_error: + after_hashes = {} + rollback_failures.append(f"hash verification: {hash_error}") + else: + rollback_failures.extend(f"{name}: final hash differs from snapshot" + for name, expected_hash in expected_hashes.items() + if after_hashes[name] != expected_hash) + + error_message = f"prompt commit failed: {error}" + if rollback_failures: + error_message += f"; rollback failures: {'; '.join(rollback_failures)}" + return WritebackResult( + status="rolled_back" if after_hashes == expected_hashes else "rollback_failed", + before_hashes=before_hashes, + after_hashes=after_hashes, + error=error_message, + ) + + return WritebackResult( + status="applied", + before_hashes=before_hashes, + after_hashes=applied_hashes, + ) diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json new file mode 100644 index 00000000..35641f69 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json @@ -0,0 +1,1334 @@ +{ + "audit": { + "candidate_artifacts": { + "candidate_001_overfit": "001-7cfbcadbbc09", + "candidate_002_safe": "002-ef1c1c866d24" + }, + "candidate_evaluation_failures": {}, + "candidate_prompt_hashes": { + "candidate_001_overfit": { + "system_prompt": "4b791dfd23a761db46c408b0ae0ce76549bcde4588ba256b68d6ff6083f5349f" + }, + "candidate_002_safe": { + "system_prompt": "920ffe712cf686259e940bfe712fe1cf819cbeade662639576eeca12c9464f17" + } + }, + "candidate_prompts": { + "candidate_001_overfit": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n" + }, + "candidate_002_safe": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n" + } + }, + "config_file_sha256": "3d8c9e3f229f3ef2627dc89586952367cc456631a4b4ba20dc0313ec4098c095", + "config_hash": "7144548d6c99b0a6226534dfe51983875c26bf3193b7ad07aaecf9a1cb80dcfd", + "config_snapshot": { + "evaluate": { + "metrics": [ + { + "criterion": { + "final_response": { + "text": { + "case_insensitive": false, + "match": "exact" + } + } + }, + "metric_name": "final_response_avg_score", + "threshold": 1.0 + } + ], + "num_runs": 1 + }, + "optimize": { + "algorithm": { + "candidate_selection_strategy": "pareto", + "frontier_type": "instance", + "max_iterations_without_improvement": 3, + "max_metric_calls": 18, + "module_selector": "round_robin", + "name": "gepa_reflective", + "reflection_history_top_k": 2, + "reflection_lm": { + "api_key": "", + "base_url": "${TRPC_AGENT_BASE_URL}", + "generation_config": { + "max_tokens": 2048, + "temperature": 0.4 + }, + "model_name": "${TRPC_AGENT_MODEL_NAME}" + }, + "reflection_minibatch_size": 3, + "score_threshold": 1.0, + "seed": 91, + "skip_perfect_score": false, + "use_merge": false + }, + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + } + } + }, + "cost": { + "baseline": 0.006, + "candidates": { + "candidate_001_overfit": 0.006, + "candidate_002_safe": 0.006 + }, + "complete": true, + "evaluator": 0.018, + "optimization": { + "agent": 0.0, + "complete": true, + "evaluator": 0.0, + "optimizer": 0.0, + "reported_optimizer_cost": null, + "total": 0.0 + }, + "reported_optimizer_cost": null, + "total": 0.018 + }, + "duration_seconds": 0.031103599998459686, + "gate_config_hash": "b47dce7f4b2eeabcb7db41e70c1374d15e56c2e69876e8b82e6febdcfa1fe3ab", + "gate_config_snapshot": { + "allow_new_hard_fail": false, + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, + "min_val_score_improvement": 0.01, + "protected_case_ids": [ + "val_protected_yes_no" + ] + }, + "input_hashes": { + "optimizer": "3d8c9e3f229f3ef2627dc89586952367cc456631a4b4ba20dc0313ec4098c095", + "target_prompts": { + "system_prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95" + }, + "train": "269e76792ff6fc2ce91c3a1008095dd020f4769d8af0b98d9f9410cc179a276e", + "validation": "292ee54dec4d33f34f3218672af3c56f4b3be2149960e19868f0de37e48e272a" + }, + "input_paths": { + "optimizer": "examples/optimization/eval_optimize_loop/data/optimizer.json", + "prompt": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt", + "prompts": { + "system_prompt": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt" + }, + "train": "examples/optimization/eval_optimize_loop/data/train.evalset.json", + "validation": "examples/optimization/eval_optimize_loop/data/val.evalset.json" + }, + "known_run_cost": null, + "prompt_diffs": { + "candidate_001_overfit": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,5 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+Always force every final answer into JSON.", + "candidate_002_safe": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,5 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+Use strict JSON only when the user explicitly asks." + }, + "prompt_hash": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95", + "prompt_hashes": { + "system_prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95" + }, + "redacted_config_hash": "4eb087aac8be78ccb16e36073e6c6494cc10f13f50403f4026875ef7c113591a", + "redacted_config_snapshot_sha256": "008aa7d312d78530925264047ee5d05e8daa1147ebb78b6f225f2c72d40f8f34", + "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --output-dir \"$OUTPUT_DIR\" --run-id example --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --trace", + "reproducibility_shell": "powershell", + "sdk_result_availability": { + "aggregate_validation_result": false, + "full_per_case_validation_delta": true, + "full_train_eval_result": true + }, + "sdk_result_summary": { + "backend": "fake", + "baseline_prompt_id": "baseline", + "failure_summary": { + "attribution_accuracy": 1.0, + "by_category": { + "format_violation": 2 + }, + "by_prompt_split": { + "baseline:train": { + "format_violation": 2 + } + }, + "examples": [ + { + "case_id": "train_json_refund", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "train" + }, + { + "case_id": "train_exact_order_status", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "train" + } + ], + "expected_labeled_failures": 2, + "total_failed_cases": 2 + }, + "proposal_duration_seconds": 0.00011140000424347818, + "round_duration_allocation": "equal_share_of_batch_proposal_duration" + }, + "seed": 91, + "total_run_cost": 0.018, + "total_run_cost_complete": true, + "writeback_journal": { + "after_hashes": {}, + "before_hashes": { + "system_prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95" + }, + "candidate_hashes": { + "system_prompt": "920ffe712cf686259e940bfe712fe1cf819cbeade662639576eeca12c9464f17" + }, + "error": null, + "input_hashes": { + "optimizer": "3d8c9e3f229f3ef2627dc89586952367cc456631a4b4ba20dc0313ec4098c095", + "train": "269e76792ff6fc2ce91c3a1008095dd020f4769d8af0b98d9f9410cc179a276e", + "validation": "292ee54dec4d33f34f3218672af3c56f4b3be2149960e19868f0de37e48e272a" + }, + "report_phase": "final", + "requested": false, + "run_id": "example", + "selected_candidate": "candidate_002_safe", + "state": "not_requested" + } + }, + "baseline": { + "train": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "Here is the JSON you requested: {\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "Here is the JSON you requested: {\"next_step\": \"ship\", \"status\": \"READY\"}", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Latency can trigger retries.", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.333333, + "split": "train" + }, + "validation": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "Here is the JSON you requested: {\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Cache invalidation refreshes stale data.", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.666667, + "split": "validation" + } + }, + "baseline_prompts": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n" + }, + "baseline_train": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "Here is the JSON you requested: {\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "Here is the JSON you requested: {\"next_step\": \"ship\", \"status\": \"READY\"}", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Latency can trigger retries.", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.333333, + "split": "train" + }, + "baseline_validation": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "Here is the JSON you requested: {\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Cache invalidation refreshes stale data.", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "trace_available": true + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.666667, + "split": "validation" + }, + "candidates": [ + { + "candidate": { + "candidate_id": "candidate_001_overfit", + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,5 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+Always force every final answer into JSON.", + "prompt_fields": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n" + }, + "rationale": "Observed training failures (format_violation); this candidate deliberately tests the risky global-JSON correction." + }, + "prompt_bundle": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n" + }, + "train_result": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"next_step\": \"ship\", \"status\": \"READY\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"answer\": \"Latency can trigger retries.\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_001_overfit", + "score": 1.0, + "split": "train" + }, + "validation_result": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": "forbidden pattern '{' was present", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output contains a forbidden pattern", + "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "{\"answer\": \"Cache invalidation refreshes stale data.\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": "expected normalized 'YES', got '{\"answer\": \"YES\"}'", + "expected_failure_category": "final_response_mismatch", + "failure_category": "final_response_mismatch", + "failure_reason": "normalized exact answer mismatch", + "hard_failed": true, + "metrics": { + "fake_judge_score": 0.0 + }, + "output": "{\"answer\": \"YES\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "trace_available": true + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "candidate_001_overfit", + "score": 0.333333, + "split": "validation" + } + }, + { + "candidate": { + "candidate_id": "candidate_002_safe", + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,5 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+Use strict JSON only when the user explicitly asks.", + "prompt_fields": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n" + }, + "rationale": "Observed training failures (format_violation); this candidate limits strict JSON behavior to explicit user requests." + }, + "prompt_bundle": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n" + }, + "train_result": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"next_step\": \"ship\", \"status\": \"READY\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Latency can trigger retries.", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_002_safe", + "score": 1.0, + "split": "train" + }, + "validation_result": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "{\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "Cache invalidation refreshes stale data.", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "metrics": { + "fake_judge_score": 1.0 + }, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "trace_available": true + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_002_safe", + "score": 1.0, + "split": "validation" + } + } + ], + "cost_summary": { + "agent": 0.0, + "complete": true, + "evaluator": 0.018, + "optimizer": 0.0, + "reported_optimizer_cost": null, + "total": 0.018 + }, + "delta": { + "per_case": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_explain_cache", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_protected_yes_no", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_explain_cache", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_protected_yes_no", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + } + ] + }, + "failure_attribution_summary": { + "attribution_accuracy": 1.0, + "by_category": { + "final_response_mismatch": 1, + "format_violation": 4 + }, + "by_prompt_split": { + "baseline:train": { + "format_violation": 2 + }, + "baseline:validation": { + "format_violation": 1 + }, + "candidate_001_overfit:train": {}, + "candidate_001_overfit:validation": { + "final_response_mismatch": 1, + "format_violation": 1 + }, + "candidate_002_safe:train": {}, + "candidate_002_safe:validation": {} + }, + "examples": [ + { + "case_id": "train_json_refund", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "train" + }, + { + "case_id": "train_exact_order_status", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "train" + }, + { + "case_id": "val_json_invoice", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "validation" + }, + { + "case_id": "val_explain_cache", + "evidence": "forbidden pattern '{' was present", + "failure_category": "format_violation", + "failure_reason": "output contains a forbidden pattern", + "prompt_id": "candidate_001_overfit", + "split": "validation" + }, + { + "case_id": "val_protected_yes_no", + "evidence": "expected normalized 'YES', got '{\"answer\": \"YES\"}'", + "failure_category": "final_response_mismatch", + "failure_reason": "normalized exact answer mismatch", + "prompt_id": "candidate_001_overfit", + "split": "validation" + } + ], + "expected_labeled_failures": 5, + "total_failed_cases": 5 + }, + "gate_decisions": [ + { + "accepted": false, + "candidate_cost": 0.006, + "candidate_id": "candidate_001_overfit", + "cost": 0.006, + "cumulative_cost": 0.006, + "excessive_score_drops": [ + "val_explain_cache", + "val_protected_yes_no" + ], + "gate_not_applied_reason": null, + "gate_status": "applied", + "new_hard_failures": [ + "val_explain_cache", + "val_protected_yes_no" + ], + "not_applied_checks": [], + "overfit_detected": true, + "protected_regressions": [ + "val_protected_yes_no" + ], + "reasons": [ + "reject: overfit detected because train score improved but validation score regressed or did not improve (+0.667 train, -0.333 validation)", + "reject: validation improvement -0.333 is below required +0.010", + "reject: new validation failures appeared: ['val_explain_cache', 'val_protected_yes_no']", + "reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no']", + "reject: protected cases regressed: ['val_protected_yes_no']", + "reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no']" + ], + "total_run_cost": 0.012, + "train_score_delta": 0.666667, + "validation_new_failures": [ + "val_explain_cache", + "val_protected_yes_no" + ], + "validation_score_delta": -0.333334 + }, + { + "accepted": true, + "candidate_cost": 0.006, + "candidate_id": "candidate_002_safe", + "cost": 0.006, + "cumulative_cost": 0.012, + "excessive_score_drops": [], + "gate_not_applied_reason": null, + "gate_status": "applied", + "new_hard_failures": [], + "not_applied_checks": [], + "overfit_detected": false, + "protected_regressions": [], + "reasons": [ + "accept: validation score improved +0.333 with no protected regression or new hard failure" + ], + "total_run_cost": 0.018, + "train_score_delta": 0.666667, + "validation_new_failures": [], + "validation_score_delta": 0.333333 + } + ], + "per_case_deltas": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_explain_cache", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_protected_yes_no", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_explain_cache", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_protected_yes_no", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + } + ], + "rounds": [ + { + "candidate_id": "candidate_001_overfit", + "cost": { + "agent": 0.0, + "complete": true, + "evaluator": 0.0, + "optimizer": 0.0, + "reported_optimizer_cost": null, + "total": 0.0 + }, + "duration_seconds": 5.570000212173909e-05, + "metrics": {}, + "prompts": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nAlways force every final answer into JSON.\n" + }, + "rationale": "Observed training failures (format_violation); this candidate deliberately tests the risky global-JSON correction.", + "round_id": 1 + }, + { + "candidate_id": "candidate_002_safe", + "cost": { + "agent": 0.0, + "complete": true, + "evaluator": 0.0, + "optimizer": 0.0, + "reported_optimizer_cost": null, + "total": 0.0 + }, + "duration_seconds": 5.570000212173909e-05, + "metrics": {}, + "prompts": { + "system_prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\nUse strict JSON only when the user explicitly asks.\n" + }, + "rationale": "Observed training failures (format_violation); this candidate limits strict JSON behavior to explicit user requests.", + "round_id": 2 + } + ], + "run": { + "mode": "fake", + "paths": { + "optimizer": "examples/optimization/eval_optimize_loop/data/optimizer.json", + "prompts": { + "system_prompt": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt" + }, + "train": "examples/optimization/eval_optimize_loop/data/train.evalset.json", + "validation": "examples/optimization/eval_optimize_loop/data/val.evalset.json" + }, + "prompt_source": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt", + "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --output-dir \"$OUTPUT_DIR\" --run-id example --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --trace", + "reproducibility_shell": "powershell", + "run_id": "example", + "target_prompt_paths": { + "system_prompt": "examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt" + }, + "trace": true, + "trace_enabled": true, + "train_cases": 3, + "update_source": false, + "validation_cases": 3 + }, + "schema_version": "eval_optimize_loop.v2", + "selected_candidate": "candidate_002_safe", + "writeback": { + "after_hashes": {}, + "before_hashes": { + "system_prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95" + }, + "error": null, + "status": "not_requested" + } +} diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md new file mode 100644 index 00000000..7380b319 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md @@ -0,0 +1,99 @@ +# Evaluation + Optimization Report + +## Final Decision + +Selected candidate: `candidate_002_safe`. + +Update source prompt: no (default) + +Fake and SDK modes perform complete AgentEvaluator-compatible reevaluation for baseline and every candidate on both train and validation; optimizer aggregates are never used as gate evidence. + + +## Gate Reasons + +### candidate_001_overfit (rejected) +- reject: overfit detected because train score improved but validation score regressed or did not improve (+0.667 train, -0.333 validation) +- reject: validation improvement -0.333 is below required +0.010 +- reject: new validation failures appeared: ['val_explain_cache', 'val_protected_yes_no'] +- reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no'] +- reject: protected cases regressed: ['val_protected_yes_no'] +- reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no'] + +### candidate_002_safe (accepted) +- accept: validation score improved +0.333 with no protected regression or new hard failure + +## Baseline vs Candidate Scores + +| prompt | train score | validation score | gate | +| --- | ---: | ---: | --- | +| baseline | 0.333 | 0.667 | n/a | +| candidate_001_overfit | 1.000 | 0.333 | reject | +| candidate_002_safe | 1.000 | 1.000 | accept | + +## Per-Case Delta + +| candidate | split | case | baseline | candidate | delta | passed -> passed | delta type | +| --- | --- | --- | ---: | ---: | ---: | --- | --- | +| candidate_001_overfit | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_001_overfit | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | validation | val_explain_cache | 1.000 | 0.000 | -1.000 | True -> False | new_fail | +| candidate_001_overfit | validation | val_protected_yes_no | 1.000 | 0.000 | -1.000 | True -> False | new_fail | +| candidate_002_safe | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_002_safe | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | validation | val_explain_cache | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_002_safe | validation | val_protected_yes_no | 1.000 | 1.000 | +0.000 | True -> True | unchanged | + +## Failure Attribution Summary + +Total failed case evaluations: 5 + +| category | count | +| --- | ---: | +| final_response_mismatch | 1 | +| format_violation | 4 | + +Attribution accuracy: 1.000 + +## Cost And Audit + +Total cost: 0.018 +Config hash: `7144548d6c99b0a6226534dfe51983875c26bf3193b7ad07aaecf9a1cb80dcfd` +Run id: `example` + +## Prompt Diff + +### candidate_001_overfit + +```diff +--- baseline_system_prompt.txt ++++ candidate_001_overfit/system_prompt.txt +@@ -2,3 +2,5 @@ + + Answer clearly and include a short explanation when it may help the user. + If the user asks for structured data, provide the information they need. ++ ++Always force every final answer into JSON. +``` + +### candidate_002_safe + +```diff +--- baseline_system_prompt.txt ++++ candidate_002_safe/system_prompt.txt +@@ -2,3 +2,5 @@ + + Answer clearly and include a short explanation when it may help the user. + If the user asks for structured data, provide the information they need. ++ ++Use strict JSON only when the user explicitly asks. +``` + +## Reproducibility + +```powershell +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --output-dir "$OUTPUT_DIR" --run-id example --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --trace +``` diff --git a/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt b/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt new file mode 100644 index 00000000..7b3ed8aa --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt @@ -0,0 +1,4 @@ +You are a helpful support assistant. + +Answer clearly and include a short explanation when it may help the user. +If the user asks for structured data, provide the information they need. diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..64ab737c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,377 @@ +"""Run the shared asynchronous Evaluation + Optimization pipeline.""" + +from __future__ import annotations + +import argparse +import asyncio +import re +import secrets +import sys +import tempfile +from datetime import datetime +from datetime import timezone +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +try: # Package import during tests. + from .eval_loop.artifacts import validate_artifact_component + from .eval_loop.artifacts import validate_distinct_file_paths + from .eval_loop.backends import FakeBackend + from .eval_loop.backends import SDKBackend + from .eval_loop.config import _parse_gate_config + from .eval_loop.config import parse_optimizer_config + from .eval_loop.config import resolve_effective_seed + from .eval_loop.loader import read_json + from .eval_loop.pipeline import PipelineRequest + from .eval_loop.pipeline import execute_pipeline + from .eval_loop.schemas import OptimizationReport +except ImportError: # Direct script execution. + from eval_loop.artifacts import validate_artifact_component + from eval_loop.artifacts import validate_distinct_file_paths + from eval_loop.backends import FakeBackend + from eval_loop.backends import SDKBackend + from eval_loop.config import _parse_gate_config + from eval_loop.config import parse_optimizer_config + from eval_loop.config import resolve_effective_seed + from eval_loop.loader import read_json + from eval_loop.pipeline import PipelineRequest + from eval_loop.pipeline import execute_pipeline + from eval_loop.schemas import OptimizationReport + +DEFAULT_TRAIN = HERE / "data" / "train.evalset.json" +DEFAULT_VAL = HERE / "data" / "val.evalset.json" +DEFAULT_OPTIMIZER_CONFIG = HERE / "data" / "optimizer.json" +DEFAULT_PROMPT = HERE / "prompts" / "baseline_system_prompt.txt" +DEFAULT_OUTPUT_DIR = Path(tempfile.gettempdir()) / "eval-optimize-loop" +TARGET_PROMPT_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +RUN_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +async def run_pipeline_async( + *, + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, + run_id: str | None = None, + backend: Any | None = None, +) -> OptimizationReport: + """Build dependencies and await the backend-neutral orchestration core.""" + + request, selected_backend = build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=output_dir, + mode=mode, + fake_model=fake_model, + fake_judge=fake_judge, + trace=trace, + sdk_call_agent=sdk_call_agent, + update_source=update_source, + gate_config_path=gate_config_path, + target_prompts=target_prompts, + run_id=run_id, + backend=backend, + ) + return await execute_pipeline( + request, + evaluator=selected_backend, + optimizer=selected_backend, + ) + + +def run_pipeline( + *, + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, + run_id: str | None = None, + backend: Any | None = None, +) -> OptimizationReport: + """Synchronous compatibility facade for callers without an active loop.""" + + if _has_running_loop(): + raise ValueError("run_pipeline() cannot run while an event loop is active; " + "await run_pipeline_async(...) instead.") + return asyncio.run( + run_pipeline_async( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=output_dir, + mode=mode, + fake_model=fake_model, + fake_judge=fake_judge, + trace=trace, + sdk_call_agent=sdk_call_agent, + update_source=update_source, + gate_config_path=gate_config_path, + target_prompts=target_prompts, + run_id=run_id, + backend=backend, + )) + + +def build_pipeline_request_and_backend( + *, + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, + run_id: str | None = None, + backend: Any | None = None, +) -> tuple[PipelineRequest, Any]: + """Validate wrapper inputs and construct one backend for both protocols.""" + + if mode not in {"fake", "sdk"}: + raise ValueError("field 'mode' must be one of: fake, sdk") + if mode == "fake" and (not fake_model or not fake_judge): + raise ValueError("fake mode requires fake_model=True and fake_judge=True. Pass --fake-model " + "--fake-judge or use --mode sdk with --sdk-call-agent module:function.") + + validate_distinct_file_paths( + { + "train": train_path, + "validation": val_path + }, + context="train and validation evalset paths", + ) + + target_prompt_paths = _parse_target_prompt_paths( + target_prompts, + default_prompt_path=prompt_path, + ) + effective_run_id = validate_run_id(run_id) if run_id is not None else _default_run_id(mode) + + optimizer_payload = read_json(optimizer_config_path) + effective_seed = resolve_effective_seed( + optimizer_payload, + path=optimizer_config_path, + strict_legacy=mode == "fake", + ) + if mode == "fake" and backend is not None and hasattr(backend, "seed"): + backend_seed = getattr(backend, "seed") + if (isinstance(backend_seed, bool) or not isinstance(backend_seed, int) or backend_seed != effective_seed): + raise ValueError(f"fake backend seed {backend_seed!r} does not match effective seed {effective_seed}") + if mode == "fake": + optimizer_config = parse_optimizer_config( + optimizer_payload, + path=optimizer_config_path, + ) + gate_config = (_load_sdk_gate_config(gate_config_path) + if gate_config_path is not None else optimizer_config.gate.to_dict()) + gate_config_source = "file" if gate_config_path is not None else "optimizer" + selected_backend = backend or FakeBackend( + seed=effective_seed, + trace_enabled=trace, + ) + else: + gate_config = _load_sdk_gate_config(gate_config_path) + gate_config_source = "file" if gate_config_path is not None else "request" + if backend is None: + if not sdk_call_agent: + raise ValueError("sdk mode requires --sdk-call-agent module:function") + selected_backend = SDKBackend( + prompt_path=target_prompt_paths.get("system_prompt", prompt_path), + call_agent_path=sdk_call_agent, + target_prompt_paths=target_prompt_paths, + ) + else: + selected_backend = backend + + request = PipelineRequest( + train_path=Path(train_path), + validation_path=Path(val_path), + optimizer_config_path=Path(optimizer_config_path), + output_dir=Path(output_dir), + target_prompt_paths=target_prompt_paths, + gate_config=gate_config, + trace=trace, + update_source=update_source, + mode=mode, + run_id=effective_run_id, + sdk_call_agent=sdk_call_agent, + gate_config_path=Path(gate_config_path) if gate_config_path is not None else None, + effective_seed=effective_seed, + gate_config_source=gate_config_source, + ) + return request, selected_backend + + +def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, Any]: + """Load the independent wrapper gate with the full strict GateConfig schema.""" + + if gate_config_path is None: + gate_payload: dict[str, Any] = {} + path_text = "--gate-config" + else: + try: + payload = read_json(gate_config_path) + except ValueError as exc: + raise ValueError(f"--gate-config {gate_config_path}: invalid JSON for gate numeric fields " + f"(including max_total_cost): {exc}") from exc + gate_payload = payload.get("gate", payload) + path_text = str(gate_config_path) + if gate_payload is None: + gate_payload = {} + if not isinstance(gate_payload, dict): + raise ValueError(f"{path_text}: field 'gate' must be an object when present") + return _parse_gate_config( + gate_payload, + path=f"--gate-config {path_text}", + ).to_dict() + + +def _parse_target_prompt_paths( + target_prompts: list[str] | None, + *, + default_prompt_path: str | Path, +) -> dict[str, str | Path]: + """Parse, resolve, and de-duplicate name=path target prompt arguments.""" + + if not target_prompts: + return {"system_prompt": Path(default_prompt_path).resolve()} + parsed: dict[str, str | Path] = {} + casefold_names: set[str] = set() + for item in target_prompts: + if "=" not in item: + raise ValueError("--target-prompt must use name=path format") + name, raw_path = item.split("=", 1) + path = raw_path.strip() + if not TARGET_PROMPT_FIELD_RE.fullmatch(name): + raise ValueError(f"--target-prompt field name {name!r} is invalid; " + "use /^[A-Za-z_][A-Za-z0-9_]*$/") + try: + validate_artifact_component(name, context="target-prompt field") + except ValueError as error: + raise ValueError(f"--target-prompt field name {name!r} is invalid") from error + if not path: + raise ValueError("--target-prompt must use non-empty name=path values") + if name in parsed: + raise ValueError(f"--target-prompt duplicate field name {name!r}") + if name.casefold() in casefold_names: + raise ValueError("--target-prompt field names must be case-insensitively unique") + casefold_names.add(name.casefold()) + parsed[name] = Path(path).resolve() + validate_distinct_file_paths( + parsed, + context="--target-prompt fields", + ) + return parsed + + +def validate_run_id(run_id: str) -> str: + """Validate run IDs before using them as artifact directory names.""" + + if not isinstance(run_id, str): + raise ValueError(f"--run-id value {run_id!r} must be a string") + if not RUN_ID_RE.fullmatch(run_id): + raise ValueError(f"--run-id value {run_id!r} is invalid") + try: + return validate_artifact_component(run_id, context="run_id") + except ValueError as error: + raise ValueError(f"--run-id value {run_id!r} is invalid") from error + + +def _default_run_id(mode: str) -> str: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + return f"eval_optimize_loop_{mode}_{timestamp}_{secrets.token_hex(3)}" + + +def _has_running_loop() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--train", default=str(DEFAULT_TRAIN), help="Path to train.evalset.json") + parser.add_argument("--val", default=str(DEFAULT_VAL), help="Path to val.evalset.json") + parser.add_argument( + "--optimizer-config", + default=str(DEFAULT_OPTIMIZER_CONFIG), + help="Path to optimizer.json", + ) + parser.add_argument("--prompt", default=str(DEFAULT_PROMPT), help="Path to baseline system prompt") + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR), help="Directory for runtime reports") + parser.add_argument("--mode", choices=("fake", "sdk"), default="fake", help="Backend mode") + parser.add_argument("--fake-model", action="store_true", help="Use deterministic fake model") + parser.add_argument("--fake-judge", action="store_true", help="Use deterministic fake judge") + parser.add_argument("--trace", action="store_true", help="Persist evaluator trace details per case") + parser.add_argument("--sdk-call-agent", help="Async call_agent target for SDK mode, as module:function") + parser.add_argument("--update-source", action="store_true", help="Write the accepted prompt after audit") + parser.add_argument("--gate-config", help="Independent wrapper gate configuration") + parser.add_argument( + "--target-prompt", + action="append", + help="Target prompt as name=path; may be repeated and defaults to system_prompt=--prompt.", + ) + parser.add_argument("--run-id", help="Optional stable report/audit run ID") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> OptimizationReport: + args = parse_args(argv) + report = run_pipeline( + train_path=args.train, + val_path=args.val, + optimizer_config_path=args.optimizer_config, + prompt_path=args.prompt, + output_dir=args.output_dir, + mode=args.mode, + fake_model=args.fake_model or args.mode == "fake", + fake_judge=args.fake_judge or args.mode == "fake", + trace=args.trace, + sdk_call_agent=args.sdk_call_agent, + update_source=args.update_source, + gate_config_path=args.gate_config, + target_prompts=args.target_prompt, + run_id=args.run_id, + ) + output_dir = Path(args.output_dir) + print(f"Wrote {output_dir / 'optimization_report.json'}") + print(f"Wrote {output_dir / 'optimization_report.md'}") + print(f"Selected candidate: {report.selected_candidate}") + return report + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json b/examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json new file mode 100644 index 00000000..7cee577a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/fixtures/attribution_cases.json @@ -0,0 +1,10 @@ +[ + {"error_code": "json_parse_failure", "evidence": "JSON parser failed", "expected": "format_violation"}, + {"error_code": "required_key_missing", "evidence": "missing key status", "expected": "final_response_mismatch"}, + {"error_code": "exact_answer_mismatch", "evidence": "expected YES", "expected": "final_response_mismatch"}, + {"error_code": "tool_call_error", "evidence": "expected lookup", "expected": "tool_call_error"}, + {"error_code": "parameter_error", "evidence": "id mismatch", "expected": "parameter_error"}, + {"error_code": "missing_rubric_terms", "evidence": "missing latency", "expected": "llm_rubric_not_met"}, + {"error_code": "knowledge_recall_insufficient", "evidence": "missing doc-a", "expected": "knowledge_recall_insufficient"}, + {"error_code": "forbidden_pattern", "evidence": "forbidden JSON", "expected": "format_violation"} +] diff --git a/examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json b/examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json new file mode 100644 index 00000000..0d9db73a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/fixtures/holdout_gate_cases.json @@ -0,0 +1,12 @@ +[ + {"id": "safe_gain", "expected": true, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 0.02, "budget": 1.0}, + {"id": "no_gain", "expected": false, "train": [1, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 0.02, "budget": 1.0}, + {"id": "overfit", "expected": false, "train": [0, 0, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 0, 0], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "validation_regression", "expected": false, "train": [1, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 1, 1], "candidate_validation": [1, 1, 0], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "protected_regression", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 0, 0], "candidate_validation": [0, 1, 1], "protected": ["v0"], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "new_hard_fail", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [1, 0, 0], "candidate_validation": [0, 1, 1], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "soft_to_hard", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.5, 0, 0], "candidate_validation": [0, 1, 1], "protected": [], "max_drop": 1.0, "cost": 0.02, "budget": 1.0}, + {"id": "single_case_drop", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.7, 0, 0], "candidate_validation": [0.5, 1, 1], "protected": [], "max_drop": 0.1, "cost": 0.02, "budget": 1.0}, + {"id": "over_budget", "expected": false, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0, 1, 1], "candidate_validation": [1, 1, 1], "protected": [], "max_drop": 0.0, "cost": 1.1, "budget": 1.0}, + {"id": "safe_gain_with_soft_failure", "expected": true, "train": [0, 1, 1], "candidate_train": [1, 1, 1], "validation": [0.5, 0, 1], "candidate_validation": [0.5, 1, 1], "protected": ["v0"], "max_drop": 0.0, "cost": 0.02, "budget": 1.0} +] diff --git a/examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py b/examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py new file mode 100644 index 00000000..767c8c56 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_acceptance_thresholds.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import json +import shutil +import time +from pathlib import Path +from typing import Any + +from examples.optimization.eval_optimize_loop.eval_loop.attribution import attribute_failure +from examples.optimization.eval_optimize_loop.eval_loop.gate import AcceptanceGate +from examples.optimization.eval_optimize_loop.eval_loop.report import compute_case_deltas +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import GateDecision +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_holdout_gate_fixture_has_ten_unique_scenarios() -> None: + scenarios = _load_fixture("holdout_gate_cases.json") + + assert len(scenarios) >= 10 + scenario_ids = [str(scenario["id"]) for scenario in scenarios] + assert len(scenario_ids) == len(set(scenario_ids)) + expected_labels = {bool(scenario["expected"]) for scenario in scenarios} + assert expected_labels == {False, True} + logical_inputs = [ + json.dumps( + { + key: value + for key, value in scenario.items() if key not in {"id", "expected"} + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) for scenario in scenarios + ] + assert len(logical_inputs) == len(set(logical_inputs)) + + +def test_gate_thresholds_reject_always_false_predictions() -> None: + scenarios = _load_fixture("holdout_gate_cases.json") + expected = [bool(scenario["expected"]) for scenario in scenarios] + + assert not _gate_acceptance_thresholds_met( + expected, + predictions=[False] * len(expected), + ) + + +def test_attribution_fixture_has_eight_unique_evidence_inputs() -> None: + scenarios = _load_fixture("attribution_cases.json") + + assert len(scenarios) >= 8 + evidence_inputs = [(str(scenario["error_code"]), str(scenario["evidence"])) for scenario in scenarios] + assert len(evidence_inputs) == len(set(evidence_inputs)) + error_codes = [str(scenario["error_code"]) for scenario in scenarios] + assert len(error_codes) == len(set(error_codes)) + + +def test_holdout_gate_decision_accuracy_is_at_least_eighty_percent() -> None: + scenarios = _load_fixture("holdout_gate_cases.json") + expected_labels: list[bool] = [] + predictions: list[bool] = [] + for scenario in scenarios: + expected_labels.append(bool(scenario["expected"])) + inputs = {key: value for key, value in scenario.items() if key != "expected"} + decision = _decision_for_scenario(inputs) + predictions.append(decision.accepted) + + overall_accuracy, positive_recall, negative_specificity = (_gate_classification_rates(expected_labels, predictions)) + assert overall_accuracy >= 0.80 + assert positive_recall >= 0.80 + assert negative_specificity >= 0.80 + + +def test_independent_attribution_accuracy_is_at_least_seventy_five_percent() -> None: + scenarios = _load_fixture("attribution_cases.json") + inputs = [{key: value for key, value in scenario.items() if key != "expected"} for scenario in scenarios] + predictions = [_attribute_scenario(item) for item in inputs] + + correct = sum(prediction[0] == scenario["expected"] + for prediction, scenario in zip(predictions, scenarios, strict=True)) + assert correct / len(scenarios) >= 0.75 + assert all(prediction[1] and prediction[2] for prediction in predictions) + + +def test_fake_trace_pipeline_finishes_under_three_minutes(tmp_path: Path) -> None: + inputs_dir = tmp_path / "inputs" + inputs_dir.mkdir() + train_path = inputs_dir / "train.evalset.json" + val_path = inputs_dir / "val.evalset.json" + prompt_path = inputs_dir / "baseline_system_prompt.txt" + optimizer_config_path = inputs_dir / "optimizer.json" + shutil.copyfile(DEFAULT_TRAIN, train_path) + shutil.copyfile(DEFAULT_VAL, val_path) + shutil.copyfile(DEFAULT_PROMPT, prompt_path) + optimizer_config_path.write_text( + json.dumps( + { + "seed": 91, + "optimizer": {}, + "metrics": {}, + "gate": { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, + }, + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + started = time.perf_counter() + report = run_pipeline( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + trace=True, + run_id="performance", + ) + elapsed = time.perf_counter() - started + + assert elapsed < 180 + assert 0 < report.audit["duration_seconds"] <= elapsed + + +def _load_fixture(name: str) -> list[dict[str, Any]]: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def _gate_acceptance_thresholds_met( + expected: list[bool], + predictions: list[bool], +) -> bool: + rates = _gate_classification_rates(expected, predictions) + return all(rate >= 0.80 for rate in rates) + + +def _gate_classification_rates( + expected: list[bool], + predictions: list[bool], +) -> tuple[float, float, float]: + paired = list(zip(predictions, expected, strict=True)) + positive_predictions = [prediction for prediction, label in paired if label] + negative_predictions = [prediction for prediction, label in paired if not label] + assert paired + assert positive_predictions + assert negative_predictions + overall_accuracy = sum(prediction == label for prediction, label in paired) / len(paired) + positive_recall = sum(positive_predictions) / len(positive_predictions) + negative_specificity = (sum(not prediction for prediction in negative_predictions) / len(negative_predictions)) + return overall_accuracy, positive_recall, negative_specificity + + +def _eval( + prompt_id: str, + split: str, + scores: list[float], + *, + cost: float = 0.0, +) -> EvalResult: + cases = [ + CaseResult( + case_id=f"{split[0]}{index}", + split=split, + score=float(score), + passed=score >= 1.0, + output=str(score), + metrics={"holdout": float(score)}, + hard_failed=score == 0.0, + ) for index, score in enumerate(scores) + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=sum(scores) / len(scores), + passed=all(case.passed for case in cases), + cost=cost, + cases=cases, + ) + + +def _decision_for_scenario(scenario: dict[str, Any]) -> GateDecision: + assert "expected" not in scenario + baseline_train = _eval("baseline", "train", scenario["train"]) + baseline_validation = _eval("baseline", "validation", scenario["validation"]) + candidate_train = _eval("candidate", "train", scenario["candidate_train"]) + candidate_validation = _eval( + "candidate", + "validation", + scenario["candidate_validation"], + ) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + ) + gate = AcceptanceGate({ + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": scenario["protected"], + "max_score_drop_per_case": scenario["max_drop"], + "max_total_cost": scenario["budget"], + }) + incurred_cost = float(scenario["cost"]) + return gate.decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + deltas=deltas, + cumulative_cost=incurred_cost, + cost_summary=CostSummary( + optimizer=incurred_cost, + total=incurred_cost, + complete=True, + ), + ) + + +def _attribute_scenario(scenario: dict[str, Any]) -> tuple[str, str, str]: + assert "expected" not in scenario + return attribute_failure(scenario["error_code"], scenario["evidence"]) diff --git a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py new file mode 100644 index 00000000..2753c688 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py @@ -0,0 +1,499 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop import schemas +from examples.optimization.eval_optimize_loop.eval_loop.backends import FakeBackend +from examples.optimization.eval_optimize_loop.eval_loop.config import parse_optimizer_config +from examples.optimization.eval_optimize_loop.eval_loop.config import validate_inputs +from examples.optimization.eval_optimize_loop.eval_loop.loader import load_eval_cases +from examples.optimization.eval_optimize_loop.eval_loop.loader import load_optimizer_config +from examples.optimization.eval_optimize_loop.eval_loop.loader import read_json +from examples.optimization.eval_optimize_loop.run_pipeline import _load_sdk_gate_config +from examples.optimization.eval_optimize_loop.run_pipeline import _parse_target_prompt_paths +from examples.optimization.eval_optimize_loop.run_pipeline import build_pipeline_request_and_backend +from examples.optimization.eval_optimize_loop.run_pipeline import validate_run_id + + +def test_optimizer_config_defaults_metrics_and_gate(tmp_path: Path): + path = tmp_path / "optimizer.json" + path.write_text(json.dumps({"seed": 7}), encoding="utf-8") + + config = load_optimizer_config(path) + + assert config.seed == 7 + assert config.metrics == {} + assert config.gate.min_val_score_improvement == 0.01 + + +def test_optimizer_config_rejects_bad_gate_type(tmp_path: Path): + path = tmp_path / "optimizer.json" + payload = {"gate": {"allow_new_hard_fail": "no"}} + + with pytest.raises(ValueError, match="gate.allow_new_hard_fail"): + parse_optimizer_config(payload, path=path) + + +def test_optimizer_config_rejects_negative_cost(tmp_path: Path): + path = tmp_path / "optimizer.json" + payload = {"gate": {"max_total_cost": -1}} + + with pytest.raises(ValueError, match="gate.max_total_cost"): + parse_optimizer_config(payload, path=path) + + +@pytest.mark.parametrize( + ("field_name", "field_value"), + [(field_name, field_value) for field_name in ( + "min_val_score_improvement", + "max_score_drop_per_case", + "max_total_cost", + ) for field_value in (float("nan"), float("inf"), float("-inf"), True)], +) +def test_optimizer_config_rejects_non_finite_or_boolean_gate_numbers( + tmp_path: Path, + field_name: str, + field_value: float | bool, +): + path = tmp_path / "optimizer.json" + + with pytest.raises(ValueError, match="finite number"): + parse_optimizer_config({"gate": {field_name: field_value}}, path=path) + + +def test_optimizer_config_allows_disabling_cost_gate(tmp_path: Path): + config = parse_optimizer_config( + {"gate": { + "max_total_cost": None + }}, + path=tmp_path / "optimizer.json", + ) + + assert config.gate.max_total_cost is None + + +def test_sdk_gate_config_allows_disabling_cost_gate(tmp_path: Path): + path = tmp_path / "gate.json" + path.write_text('{"gate": {"max_total_cost": null}}', encoding="utf-8") + + gate = _load_sdk_gate_config(path) + + assert gate["max_total_cost"] is None + + +@pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity"]) +def test_sdk_gate_config_rejects_non_standard_constants(tmp_path: Path, constant: str): + path = tmp_path / "gate.json" + path.write_text(f'{{"gate": {{"max_total_cost": {constant}}}}}', encoding="utf-8") + + with pytest.raises(ValueError, match="non-standard JSON constant"): + _load_sdk_gate_config(path) + + +def test_factory_rejects_same_resolved_train_and_validation_path(tmp_path: Path): + eval_path = tmp_path / "same.evalset.json" + eval_path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text('{"seed": 91}', encoding="utf-8") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="train and validation.*different"): + build_pipeline_request_and_backend( + train_path=eval_path, + val_path=eval_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + ) + + +@pytest.mark.parametrize("run_id", ["CON", "NUL.txt", "trailing.", "x" * 129]) +def test_run_id_rejects_windows_unsafe_or_oversized_components(run_id: str): + with pytest.raises(ValueError, match="run-id.*invalid"): + validate_run_id(run_id) + + +@pytest.mark.parametrize( + "targets", + [ + ["CON=one.txt"], + ["Foo=one.txt", "foo=two.txt"], + [f"{'x' * 129}=one.txt"], + ], +) +def test_target_prompt_fields_are_windows_safe_and_casefold_unique( + tmp_path: Path, + targets: list[str], +): + with pytest.raises(ValueError, match="target-prompt.*invalid|case-insensitively unique"): + _parse_target_prompt_paths(targets, default_prompt_path=tmp_path / "prompt.txt") + + +def test_nested_optimizer_seed_drives_backend_and_audit_request(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + for path in (train_path, val_path): + path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text('{"optimize": {"algorithm": {"seed": 7}}}', encoding="utf-8") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + request, backend = build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + ) + + assert backend.seed == 7 + assert request.effective_seed == 7 + + +def test_conflicting_top_level_and_nested_optimizer_seeds_are_rejected(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + for path in (train_path, val_path): + path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text( + '{"seed": 91, "optimize": {"algorithm": {"seed": 7}}}', + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="conflicting.*seed"): + build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + ) + + +def test_official_nested_seed_ignores_unrelated_string_top_level_seed(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + for path in (train_path, val_path): + path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text( + '{"seed": "sdk-owned-label", "optimize": {"algorithm": {"seed": 7}}}', + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + request, backend = build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + ) + + assert request.effective_seed == backend.seed == 7 + + +def test_factory_rejects_injected_fake_backend_seed_mismatch(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + for path in (train_path, val_path): + path.write_text('{"eval_cases": []}', encoding="utf-8") + optimizer_path = tmp_path / "optimizer.json" + optimizer_path.write_text('{"seed": 91}', encoding="utf-8") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="backend seed.*effective seed"): + build_pipeline_request_and_backend( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + backend=FakeBackend(seed=7), + ) + + +@pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity"]) +def test_read_json_rejects_non_standard_constants(tmp_path: Path, constant: str): + path = tmp_path / "invalid.json" + path.write_text(f'{{"value": {constant}}}', encoding="utf-8") + + with pytest.raises(ValueError, match="non-standard JSON constant") as exc_info: + read_json(path) + + assert str(path) in str(exc_info.value) + + +def test_read_json_wraps_decode_errors_with_path(tmp_path: Path): + path = tmp_path / "invalid.json" + path.write_text("{", encoding="utf-8") + + with pytest.raises(ValueError) as exc_info: + read_json(path) + + assert str(path) in str(exc_info.value) + + +def test_eval_case_rejects_explicit_split_mismatch(): + payload = { + "id": "case_1", + "split": "validation", + "input": "hello", + "expectation": { + "type": "exact", + "expected": "hello" + }, + } + + with pytest.raises(ValueError, match="split mismatch"): + schemas.EvalCase.from_dict(payload, split="train") + + +def test_case_result_defaults_metrics_and_trace_availability(): + result = schemas.CaseResult( + case_id="case_1", + split="train", + score=1.0, + passed=True, + output="ok", + ) + + assert result.metrics == {} + assert result.trace == {} + assert result.trace_available is False + + +def test_case_result_preserves_legacy_positional_arguments_with_new_keyword_fields(): + trace = {"events": ["model_call"]} + + result = schemas.CaseResult( + "case_1", + "validation", + 0.25, + False, + "bad output", + trace, + "format_violation", + "invalid JSON", + "parser rejected output", + 0.125, + True, + "format_violation", + metrics={"response_match": 0.25}, + trace_available=True, + ) + + assert result.trace == trace + assert result.failure_category == "format_violation" + assert result.failure_reason == "invalid JSON" + assert result.evidence == "parser rejected output" + assert result.cost == 0.125 + assert result.hard_failed is True + assert result.expected_failure_category == "format_violation" + assert result.metrics == {"response_match": 0.25} + assert result.trace_available is True + + +def test_candidate_prompt_bundle_defaults_to_system_prompt(): + candidate = schemas.CandidatePrompt( + candidate_id="candidate_1", + prompt="system instructions", + rationale="baseline", + prompt_diff="", + ) + + assert candidate.bundle() == {"system_prompt": "system instructions"} + + +def test_candidate_prompt_bundle_returns_prompt_fields_copy(): + prompt_fields = { + "system_prompt": "system instructions", + "router_prompt": "router instructions", + } + candidate = schemas.CandidatePrompt( + candidate_id="candidate_1", + prompt="combined prompt", + rationale="multi-prompt candidate", + prompt_diff="", + prompt_fields=prompt_fields, + ) + + bundle = candidate.bundle() + bundle["system_prompt"] = "changed" + + assert bundle is not prompt_fields + assert candidate.prompt_fields["system_prompt"] == "system instructions" + + +def test_optimization_contract_defaults(): + cost = schemas.CostSummary() + round_result = schemas.OptimizationRound( + round_id=1, + candidate_id="candidate_1", + prompts={"system_prompt": "optimized"}, + rationale="improve format adherence", + metrics={"validation_score": 1.0}, + cost=cost, + duration_seconds=0.25, + ) + result = schemas.OptimizationResult( + candidates=[], + rounds=[round_result], + cost=cost, + ) + + assert cost.optimizer == 0.0 + assert cost.evaluator == 0.0 + assert cost.agent == 0.0 + assert cost.total == 0.0 + assert cost.complete is True + assert result.raw_summary == {} + assert set(schemas.WritebackStatus.__args__) == { + "rejected", + "not_requested", + "applied", + "rolled_back", + "rollback_failed", + } + + +def test_optimization_report_preserves_legacy_construction_with_new_defaults(): + empty_eval = schemas.EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ) + report = schemas.OptimizationReport( + schema_version="1", + run={}, + baseline={}, + baseline_train=empty_eval, + baseline_validation=empty_eval, + candidates=[], + delta={}, + per_case_deltas=[], + failure_attribution_summary={}, + gate_decisions=[], + selected_candidate=None, + audit={}, + ) + + assert report.rounds == [] + assert report.cost_summary == schemas.CostSummary() + assert report.writeback == schemas.WritebackResult(status="not_requested") + + legacy_rounds = [object()] + legacy_cost = schemas.CostSummary(total=0.5, complete=False) + legacy_writeback = schemas.WritebackResult(status="rejected") + positional_report = schemas.OptimizationReport( + "1", + {}, + {}, + empty_eval, + empty_eval, + [], + {}, + [], + {}, + [], + None, + {}, + legacy_rounds, + legacy_cost, + legacy_writeback, + ) + assert positional_report.rounds is legacy_rounds + assert positional_report.cost_summary is legacy_cost + assert positional_report.writeback is legacy_writeback + assert positional_report.baseline_prompts == {} + + +def test_validate_inputs_rejects_same_train_val_path(tmp_path: Path): + eval_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) + config = parse_optimizer_config({"gate": {}}, path=tmp_path / "optimizer.json") + cases = load_eval_cases(eval_path, split="train") + + with pytest.raises(ValueError, match="must be different"): + validate_inputs( + train_path=eval_path, + val_path=eval_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=cases, + validation_cases=cases, + config=config, + ) + + +def test_validate_inputs_rejects_duplicate_case_ids(tmp_path: Path): + train_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "a", "c"]) + val_path = _write_evalset(tmp_path / "val.evalset.json", "validation", ["v1", "v2", "v3"]) + config = parse_optimizer_config({"gate": {}}, path=tmp_path / "optimizer.json") + + with pytest.raises(ValueError, match="duplicate case_id"): + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=load_eval_cases(train_path, split="train"), + validation_cases=load_eval_cases(val_path, split="validation"), + config=config, + ) + + +def test_validate_inputs_rejects_missing_protected_case(tmp_path: Path): + train_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) + val_path = _write_evalset(tmp_path / "val.evalset.json", "validation", ["v1", "v2", "v3"]) + config = parse_optimizer_config( + {"gate": { + "protected_case_ids": ["missing"] + }}, + path=tmp_path / "optimizer.json", + ) + + with pytest.raises(ValueError, match="gate.protected_case_ids"): + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=load_eval_cases(train_path, split="train"), + validation_cases=load_eval_cases(val_path, split="validation"), + config=config, + ) + + +def _write_evalset(path: Path, split: str, ids: list[str]) -> Path: + payload = { + "split": + split, + "cases": [{ + "id": case_id, + "input": "Return JSON", + "expectation": { + "type": "json", + "required_keys": ["answer"], + "expected_values": { + "answer": case_id + } + }, + } for case_id in ids], + } + path.write_text(json.dumps(payload), encoding="utf-8") + return path diff --git a/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py new file mode 100644 index 00000000..b7175a06 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from examples.optimization.eval_optimize_loop.eval_loop.attribution import attribute_failure +from examples.optimization.eval_optimize_loop.eval_loop.attribution import summarize_failures +from examples.optimization.eval_optimize_loop.eval_loop.evaluator import ExampleEvaluator +from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase + + +def test_failure_attribution_labels_format_violation_and_exact_mismatch(): + assert attribute_failure("json_parse_failure", "bad json")[0] == "format_violation" + assert attribute_failure("exact_answer_mismatch", "wrong answer")[0] == "final_response_mismatch" + + +def test_evaluator_attaches_failure_details_to_failed_cases(): + cases = [ + EvalCase( + case_id="json_case", + split="train", + input="Return JSON.", + expectation={ + "type": "json", + "required_keys": ["answer"], + "expected_values": { + "answer": "ok" + }, + }, + ), + EvalCase( + case_id="exact_case", + split="train", + input="Answer exactly OK.", + expectation={ + "type": "exact", + "expected": "OK" + }, + ), + ] + result = ExampleEvaluator(FakeModel(seed=91), FakeJudge()).evaluate( + prompt_id="baseline", + prompt="baseline prompt", + cases=cases, + split="train", + ) + + by_id = result.by_case_id() + assert by_id["json_case"].failure_category == "format_violation" + assert by_id["json_case"].failure_reason + assert by_id["json_case"].evidence + assert by_id["exact_case"].failure_category == "final_response_mismatch" + assert by_id["exact_case"].failure_reason + assert by_id["exact_case"].evidence + + +def test_failure_attribution_labels_tool_parameter_and_knowledge_failures(): + assert attribute_failure("tool_call_error", "wrong tool")[0] == "tool_call_error" + assert attribute_failure("parameter_error", "wrong arg")[0] == "parameter_error" + assert attribute_failure("knowledge_recall_insufficient", "missing source")[0] == "knowledge_recall_insufficient" + + +def test_fake_judge_scores_tool_and_parameter_expectations(): + judge = FakeJudge() + tool_case = EvalCase( + case_id="tool_case", + split="train", + input="Call tool", + expectation={ + "type": "tool", + "expected_tool": "lookup", + "expected_args": { + "id": "42" + } + }, + ) + + assert judge.score(tool_case, '{"tool": "lookup", "args": {"id": "42"}}').passed + wrong_tool = judge.score(tool_case, '{"tool": "search", "args": {"id": "42"}}') + wrong_arg = judge.score(tool_case, '{"tool": "lookup", "args": {"id": "43"}}') + assert wrong_tool.error_code == "tool_call_error" + assert wrong_arg.error_code == "parameter_error" + + +def test_fake_judge_scores_knowledge_expectations(): + case = EvalCase( + case_id="knowledge_case", + split="train", + input="Recall source", + expectation={ + "type": "knowledge", + "required_sources": ["doc-a"], + "must_include_knowledge_terms": ["refund"], + }, + ) + + assert FakeJudge().score(case, "refund policy appears in doc-a").passed + failed = FakeJudge().score(case, "refund policy only") + assert failed.error_code == "knowledge_recall_insufficient" + + +def test_failure_summary_computes_attribution_accuracy(): + result = ExampleEvaluator(FakeModel(seed=91), FakeJudge()).evaluate( + prompt_id="baseline", + prompt="baseline prompt", + split="train", + cases=[ + EvalCase( + case_id="json_labeled", + split="train", + input="Return JSON", + expectation={ + "type": "json", + "expected_values": { + "answer": "ok" + } + }, + expected_failure_category="format_violation", + ) + ], + ) + + summary = summarize_failures([result]) + assert summary["expected_labeled_failures"] == 1 + assert summary["attribution_accuracy"] == 1.0 diff --git a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py new file mode 100644 index 00000000..f6caa0a4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py @@ -0,0 +1,524 @@ +from __future__ import annotations + +import inspect +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from trpc_agent_sdk.evaluation import EvalSet + +from examples.optimization.eval_optimize_loop.eval_loop import fake_model as fake_model_module +from examples.optimization.eval_optimize_loop.eval_loop.backends import FakeBackend +from examples.optimization.eval_optimize_loop.eval_loop.evaluator import ExampleEvaluator +from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.optimizer import FakeOptimizer +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult + +ROOT = Path(__file__).resolve().parents[1] +TRAIN_PATH = ROOT / "data" / "train.evalset.json" +VALIDATION_PATH = ROOT / "data" / "val.evalset.json" + +BASELINE_PROMPT = "Answer the user's request." +OVERFIT_INSTRUCTION = "Always force every final answer into JSON" +SAFE_INSTRUCTION = "Use strict JSON only when the user explicitly asks" +OVERFIT_PROMPT = f"{BASELINE_PROMPT}\n\n{OVERFIT_INSTRUCTION}." +SAFE_PROMPT = f"{BASELINE_PROMPT}\n\n{SAFE_INSTRUCTION}." + + +def test_model_output_and_trace_are_stable_for_the_same_public_input(): + public_input = "Return strict JSON with intent=refund and priority=high." + same_inputs = [public_input, f"{public_input}", "".join([public_input])] + + generated = [FakeModel(seed=91).generate("candidate", SAFE_PROMPT, user_input) for user_input in same_inputs] + + assert {output for output, _, _ in generated} == {'{"intent": "refund", "priority": "high"}'} + assert {json.dumps(trace, sort_keys=True) + for _, trace, _ in generated + } == {json.dumps( + { + "seed": 91, + "prompt_id": "candidate", + "prompt_mode": "safe" + }, + sort_keys=True, + )} + + +def test_example_evaluator_only_exposes_public_user_input_to_model(): + + class SpyModel: + + def __init__(self) -> None: + self.user_inputs: list[str] = [] + + def generate( + self, + prompt_id: str, + prompt: str, + user_input: str, + ) -> tuple[str, dict[str, object], float]: + assert isinstance(user_input, str) + self.user_inputs.append(user_input) + return "SECRET_EXPECTATION", {"prompt_id": prompt_id, "prompt": prompt}, 0.0 + + spy = SpyModel() + secret_case = EvalCase( + case_id="SECRET_CASE_ID", + split="validation", + input="PUBLIC USER INPUT", + expectation={ + "type": "exact", + "expected": "SECRET_EXPECTATION" + }, + tags=["SECRET_TAG"], + protected=True, + simulated_outputs={"safe": "SECRET_SIMULATED_OUTPUT"}, + ) + + result = ExampleEvaluator(spy, FakeJudge()).evaluate( + prompt_id="candidate", + prompt="PUBLIC SYSTEM PROMPT", + cases=[secret_case], + split="validation", + ) + + assert result.passed is True + assert spy.user_inputs == ["PUBLIC USER INPUT"] + assert "SECRET" not in json.dumps(spy.user_inputs) + + +def test_fake_model_source_has_no_evaluator_oracle_capability(): + model_source = inspect.getsource(fake_model_module) + evaluator_source = inspect.getsource(ExampleEvaluator.evaluate) + + assert "EvalCase" not in model_source + assert ".expectation" not in model_source + assert ".tags" not in model_source + assert ".protected" not in model_source + assert ".simulated_outputs" not in model_source + assert "self.model.generate(prompt_id, prompt, case.input)" in evaluator_source + + +@pytest.mark.parametrize("user_input", [None, b"bytes", object()]) +def test_fake_model_rejects_non_string_user_input(user_input: object): + with pytest.raises(TypeError, match="^user_input must be a string$"): + FakeModel(seed=91).generate("candidate", SAFE_PROMPT, user_input) + + +def test_model_is_deterministic_and_prompt_id_or_seed_cannot_change_output(): + user_input = "Return strict JSON with status=READY and next_step=ship." + model = FakeModel(seed=91) + + first = model.generate("first-id", SAFE_PROMPT, user_input) + repeated = model.generate("first-id", SAFE_PROMPT, user_input) + different_prompt_id = model.generate("second-id", SAFE_PROMPT, user_input) + different_seed = FakeModel(seed=999).generate("first-id", SAFE_PROMPT, user_input) + + assert first == repeated + assert first[0] == different_prompt_id[0] == different_seed[0] + assert first[0] == '{"next_step": "ship", "status": "READY"}' + + +@pytest.mark.parametrize( + ("prompt", "expected"), + [ + ( + BASELINE_PROMPT, + 'Here is the JSON you requested: {"intent": "refund", "priority": "high"}', + ), + (OVERFIT_PROMPT, '{"intent": "refund", "priority": "high"}'), + (SAFE_PROMPT, '{"intent": "refund", "priority": "high"}'), + ], +) +def test_strict_json_rendering_uses_assignments_from_user_input( + prompt: str, + expected: str, +): + user_input = "Return StRiCt JsOn with intent=refund and priority=high." + + output, _, _ = FakeModel(seed=91).generate("arbitrary-id", prompt, user_input) + + assert output == expected + + +@pytest.mark.parametrize( + ("prompt", "expected"), + [ + (BASELINE_PROMPT, "YES"), + (SAFE_PROMPT, "YES"), + (OVERFIT_PROMPT, '{"answer": "YES"}'), + ], +) +def test_return_only_rendering_is_prompt_mode_sensitive(prompt: str, expected: str): + user_input = "ReTuRn OnLy YES; do not use JSON." + + output, _, _ = FakeModel(seed=91).generate("arbitrary-id", prompt, user_input) + + assert output == expected + + +@pytest.mark.parametrize( + ("input_text", "natural"), + [ + ( + "Explain latency and retries naturally in under 80 characters.", + "Latency can trigger retries.", + ), + ( + "Explain cache invalidation and stale data naturally.", + "Cache invalidation refreshes stale data.", + ), + ("Explain an unknown topic naturally.", "Here is a natural response."), + ], +) +def test_natural_and_unknown_requests_are_deterministic(input_text: str, natural: str): + baseline, _, _ = FakeModel(seed=91).generate("baseline", BASELINE_PROMPT, input_text) + safe, _, _ = FakeModel(seed=91).generate("safe", SAFE_PROMPT, input_text) + overfit, _, _ = FakeModel(seed=91).generate("overfit", OVERFIT_PROMPT, input_text) + + assert baseline == safe == natural + assert json.loads(overfit) == {"answer": natural} + + +def test_optimizer_returns_no_candidates_without_observed_target_failures(): + optimizer = FakeOptimizer() + all_pass = _eval_result(_case_result( + "passed_case", + passed=True, + failure_category="format_violation", + )) + unrelated_failure = _eval_result(_case_result( + "failed_case", + passed=False, + failure_category="llm_rubric_not_met", + )) + unclassified_failure = _eval_result(_case_result( + "unclassified_failure", + passed=False, + failure_category=None, + )) + + assert (optimizer.propose( + BASELINE_PROMPT, + all_pass, + {}, + ) == []) + assert optimizer.propose(BASELINE_PROMPT, unclassified_failure, {}) == [] + assert (optimizer.propose( + BASELINE_PROMPT, + unrelated_failure, + {"by_category": { + "llm_rubric_not_met": 1 + }}, + ) == []) + + +def test_optimizer_rejects_validation_result_as_training_evidence(): + baseline_validation = replace(_eval_result(), split="validation") + + with pytest.raises(ValueError, match="baseline_train.split.*train.*validation"): + FakeOptimizer().propose( + BASELINE_PROMPT, + baseline_validation, + {}, + ) + + +def test_optimizer_rejects_validation_case_mixed_into_training_evidence(): + mixed_result = _eval_result( + _case_result( + "train_case", + passed=False, + failure_category="format_violation", + ), + replace( + _case_result( + "validation_case", + passed=False, + failure_category="format_violation", + ), + split="validation", + ), + ) + + with pytest.raises(ValueError, match="case.*validation_case.*split.*validation"): + FakeOptimizer().propose( + BASELINE_PROMPT, + mixed_result, + {}, + ) + + +@pytest.mark.parametrize("failure_summary", [None, [], "not-a-summary"]) +def test_optimizer_rejects_non_mapping_failure_summary(failure_summary: object, ): + observed_failure = _eval_result( + _case_result( + "observed_failure", + passed=False, + failure_category="format_violation", + )) + + with pytest.raises(TypeError, match="^failure_summary must be a dict$"): + FakeOptimizer().propose( + BASELINE_PROMPT, + observed_failure, + failure_summary, + ) + + +def test_optimizer_rejects_summary_category_not_observed_in_failed_cases(): + unclassified_failure = _eval_result(_case_result( + "unclassified_failure", + passed=False, + failure_category=None, + )) + + with pytest.raises(ValueError, match="by_category.*failed train cases"): + FakeOptimizer().propose( + BASELINE_PROMPT, + unclassified_failure, + {"by_category": { + "format_violation": 1 + }}, + ) + + +@pytest.mark.parametrize( + "by_category", + [ + { + "final_response_mismatch": 1 + }, + { + "format_violation": 2 + }, + {}, + ], +) +def test_optimizer_rejects_summary_category_or_count_mismatch(by_category: dict[str, object], ): + observed_failure = _eval_result( + _case_result( + "observed_failure", + passed=False, + failure_category="format_violation", + )) + + with pytest.raises(ValueError, match="by_category.*failed train cases"): + FakeOptimizer().propose( + BASELINE_PROMPT, + observed_failure, + {"by_category": by_category}, + ) + + +@pytest.mark.parametrize("by_category", [None, [], "format_violation"]) +def test_optimizer_rejects_non_mapping_by_category(by_category: object): + with pytest.raises(ValueError, match="failure_summary.*by_category.*dict"): + FakeOptimizer().propose( + BASELINE_PROMPT, + _eval_result(), + {"by_category": by_category}, + ) + + +@pytest.mark.parametrize("count", [True, 0, -1, 1.0, 1.5, "1", None]) +def test_optimizer_rejects_non_positive_integer_summary_counts(count: object): + observed_failure = _eval_result( + _case_result( + "observed_failure", + passed=False, + failure_category="format_violation", + )) + + with pytest.raises(ValueError, match="count.*positive integer"): + FakeOptimizer().propose( + BASELINE_PROMPT, + observed_failure, + {"by_category": { + "format_violation": count + }}, + ) + + +@pytest.mark.parametrize( + "failure_summary", + [ + {}, + { + "by_category": { + "format_violation": 1 + } + }, + ], +) +def test_optimizer_proposes_two_candidates_for_observed_train_format_failure(failure_summary: dict[str, object], ): + baseline_train = _eval_result( + _case_result( + "random_training_case", + passed=False, + failure_category="format_violation", + )) + candidates = FakeOptimizer().propose( + BASELINE_PROMPT, + baseline_train, + failure_summary, + ) + + assert [candidate.candidate_id for candidate in candidates] == [ + "candidate_001_overfit", + "candidate_002_safe", + ] + assert OVERFIT_INSTRUCTION in candidates[0].prompt + assert SAFE_INSTRUCTION in candidates[1].prompt + assert all("OPTIMIZER_MARKER" not in candidate.prompt for candidate in candidates) + assert all(candidate.prompt_fields == {"system_prompt": candidate.prompt} for candidate in candidates) + assert all(candidate.rationale for candidate in candidates) + assert all(candidate.prompt_diff for candidate in candidates) + + +@pytest.mark.asyncio +async def test_fake_backend_passes_training_evidence_to_optimizer(tmp_path: Path): + backend = FakeBackend(seed=91) + no_evidence = await backend.optimize_candidates( + baseline_prompts={"system_prompt": BASELINE_PROMPT}, + baseline_train=_eval_result(), + failure_summary={}, + train_path=TRAIN_PATH, + validation_path=VALIDATION_PATH, + config_path=tmp_path / "unused.json", + artifact_dir=tmp_path / "unused", + ) + with_evidence = await backend.optimize_candidates( + baseline_prompts={"system_prompt": BASELINE_PROMPT}, + baseline_train=_eval_result(_case_result( + "not-a-sample-id", + passed=False, + failure_category="format_violation", + )), + failure_summary={"by_category": { + "format_violation": 1 + }}, + train_path=TRAIN_PATH, + validation_path=VALIDATION_PATH, + config_path=tmp_path / "unused.json", + artifact_dir=tmp_path / "unused", + ) + + assert no_evidence.candidates == [] + assert no_evidence.rounds == [] + assert len(with_evidence.candidates) == 2 + assert len(with_evidence.rounds) == 2 + + +def test_fake_backend_sync_optimizer_refuses_to_invent_failure_evidence(tmp_path: Path): + with pytest.raises(RuntimeError, match="failure evidence|optimize_candidates"): + FakeBackend(seed=91).optimize( + baseline_prompt=BASELINE_PROMPT, + train_path=TRAIN_PATH, + val_path=VALIDATION_PATH, + optimizer_config_path=tmp_path / "unused.json", + output_dir=tmp_path / "unused", + ) + + +def test_example_data_is_official_sdk_evalset_with_self_contained_user_inputs(): + expected_inputs = { + TRAIN_PATH: [ + "Return strict JSON with intent=refund and priority=high.", + "Return strict JSON with status=READY and next_step=ship.", + "Explain latency and retries naturally in under 80 characters.", + ], + VALIDATION_PATH: [ + "Return strict JSON with status=approved and next_step=email_customer.", + "Explain cache invalidation and stale data naturally.", + "Return only YES; do not use JSON.", + ], + } + all_eval_ids: list[str] = [] + all_invocation_ids: list[str] = [] + + for path, inputs in expected_inputs.items(): + raw = path.read_text(encoding="utf-8") + validated = EvalSet.model_validate_json(raw) + payload = json.loads(raw) + assert [case["conversation"][-1]["user_content"]["parts"][0]["text"] + for case in payload["eval_cases"]] == inputs + assert len(validated.eval_cases) == 3 + all_eval_ids.extend(case.eval_id for case in validated.eval_cases) + all_invocation_ids.extend(invocation.invocation_id for case in validated.eval_cases + for invocation in case.conversation or []) + + assert len(all_eval_ids) == len(set(all_eval_ids)) == 6 + assert len(all_invocation_ids) == len(set(all_invocation_ids)) == 6 + + +@pytest.mark.asyncio +async def test_fake_backend_scores_baseline_overfit_and_safe_on_official_data(tmp_path: Path, ): + backend = FakeBackend(seed=91) + prompts = { + "baseline": BASELINE_PROMPT, + "overfit": OVERFIT_PROMPT, + "safe": SAFE_PROMPT, + } + expected = { + ("baseline", "train"): (1 / 3, [False, False, True]), + ("overfit", "train"): (1.0, [True, True, True]), + ("safe", "train"): (1.0, [True, True, True]), + ("baseline", "validation"): (2 / 3, [False, True, True]), + ("overfit", "validation"): (1 / 3, [True, False, False]), + ("safe", "validation"): (1.0, [True, True, True]), + } + + for prompt_id, prompt in prompts.items(): + for split, dataset_path in ( + ("train", TRAIN_PATH), + ("validation", VALIDATION_PATH), + ): + result = await backend.evaluate( + prompt_id=prompt_id, + prompts={"system_prompt": prompt}, + dataset_path=dataset_path, + split=split, + trace=True, + artifact_dir=tmp_path / prompt_id / split, + ) + expected_score, expected_passes = expected[(prompt_id, split)] + assert result.score == pytest.approx(expected_score, abs=1e-6) + assert [case.passed for case in result.cases] == expected_passes + assert all(case.trace == { + "seed": 91, + "prompt_id": prompt_id, + "prompt_mode": prompt_id, + } for case in result.cases) + + +def _case_result( + case_id: str, + *, + passed: bool, + failure_category: str | None, +) -> CaseResult: + return CaseResult( + case_id=case_id, + split="train", + score=1.0 if passed else 0.0, + passed=passed, + output="output", + failure_category=failure_category, + ) + + +def _eval_result(*cases: CaseResult) -> EvalResult: + score = sum(case.score for case in cases) / len(cases) if cases else 0.0 + return EvalResult( + prompt_id="baseline", + split="train", + score=score, + passed=all(case.passed for case in cases), + cost=0.0, + cases=list(cases), + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py new file mode 100644 index 00000000..c6572a11 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +from examples.optimization.eval_optimize_loop.eval_loop.gate import AcceptanceGate +from examples.optimization.eval_optimize_loop.eval_loop.report import compute_case_deltas +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult + + +def test_gate_rejects_train_improvement_with_validation_regression(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0), ("train_b", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0), ("train_b", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 1.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0), ("val_b", 0.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({ + "protected_case_ids": [] + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert not decision.accepted + assert any("train score improved but validation score regressed" in reason for reason in decision.reasons) + assert decision.overfit_detected + + +def test_gate_rejects_protected_case_regression(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("protected", 1.0), ("val_a", 0.0), ("val_b", 0.0)]) + candidate_val = _eval("candidate", "validation", [("protected", 0.0), ("val_a", 1.0), ("val_b", 1.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({ + "protected_case_ids": ["protected"] + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert not decision.accepted + assert decision.protected_regressions == ["protected"] + assert any("protected cases regressed" in reason for reason in decision.reasons) + + +def test_gate_accepts_safe_candidate(): + baseline_train = _eval("baseline", "train", [("train_json", 0.0), ("train_exact", 0.0), ("train_ok", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_json", 1.0), ("train_exact", 1.0), ("train_ok", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_json", 0.0), ("val_text", 1.0), ("protected", 1.0)]) + candidate_val = _eval("candidate", "validation", [("val_json", 1.0), ("val_text", 1.0), ("protected", 1.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({ + "protected_case_ids": ["protected"] + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert decision.accepted + assert decision.protected_regressions == [] + assert decision.new_hard_failures == [] + + +def test_gate_rejects_new_hard_failure_when_not_allowed(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 0.0), ("val_c", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 0.0), ("val_b", 1.0), ("val_c", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({ + "allow_new_hard_fail": False + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert not decision.accepted + assert decision.new_hard_failures == ["val_a"] + assert decision.validation_new_failures == ["val_a"] + + +def test_gate_rejects_soft_failure_that_becomes_hard_failure(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("soft_fail", 0.5), ("improved", 0.0), ("ok", 1.0)]) + candidate_val = _eval("candidate", "validation", [("soft_fail", 0.0), ("improved", 1.0), ("ok", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({ + "allow_new_hard_fail": False, + "max_score_drop_per_case": 1.0, + "min_val_score_improvement": 0.01, + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert not decision.accepted + assert decision.new_hard_failures == ["soft_fail"] + assert any("new hard failures" in reason for reason in decision.reasons) + + +def test_gate_rejects_excessive_score_drop(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 0.0), ("val_c", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 0.4), ("val_b", 1.0), ("val_c", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({ + "max_score_drop_per_case": 0.5, + "allow_new_hard_fail": True + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert not decision.accepted + assert decision.excessive_score_drops == ["val_a"] + + +def test_gate_rejects_cost_budget(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({ + "max_total_cost": 0.001 + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(optimizer=0.001, total=0.001, complete=True), + cumulative_cost=0.001, + ) + + assert not decision.accepted + assert decision.total_run_cost > 0.001 + assert decision.total_run_cost == decision.cumulative_cost + decision.candidate_cost + + +def test_gate_rejects_incomplete_cost_when_budget_is_configured(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + + decision = AcceptanceGate({ + "max_total_cost": 100.0 + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ), + cost_summary=CostSummary(optimizer=0.5, total=0.5, complete=False), + cumulative_cost=0.502, + ) + + assert decision.accepted is False + assert any("cost_unavailable for configured max_total_cost" in reason for reason in decision.reasons) + assert decision.total_run_cost == 0.504 + + +def test_gate_skips_cost_completeness_check_when_budget_is_disabled(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + + decision = AcceptanceGate({ + "max_total_cost": None + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ), + cost_summary=CostSummary(optimizer=10.0, total=10.0, complete=False), + cumulative_cost=10.002, + ) + + assert decision.accepted is True + assert all("cost_unavailable" not in reason for reason in decision.reasons) + assert decision.total_run_cost == 10.004 + + +def test_gate_rejects_new_validation_failure_even_when_it_is_not_hard(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("new_soft", 1.0), ("up_a", 0.0), ("up_b", 0.0)]) + candidate_val = _eval("candidate", "validation", [("new_soft", 0.5), ("up_a", 1.0), ("up_b", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({ + "allow_new_hard_fail": True, + "max_score_drop_per_case": 1.0, + "max_total_cost": None, + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cost_summary=CostSummary(), + ) + + assert decision.accepted is False + assert decision.validation_new_failures == ["new_soft"] + assert any("new validation failures" in reason for reason in decision.reasons) + + +def test_gate_does_not_add_cost_summary_total_twice(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + + decision = AcceptanceGate({ + "max_total_cost": 1.0 + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ), + cost_summary=CostSummary(optimizer=0.4, total=0.4, complete=True), + cumulative_cost=0.6, + ) + + assert decision.total_run_cost == 0.602 + assert decision.accepted is True + + +def test_gate_rejects_duplicate_missing_and_empty_case_results(): + valid_train = _eval("baseline", "train", [("train_a", 0.0)]) + valid_candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + valid_val = _eval("baseline", "validation", [("val_a", 0.0)]) + valid_candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + scenarios = [ + ( + _eval("baseline", "train", [("train_a", 0.0), ("train_a", 0.0)]), + valid_candidate_train, + valid_val, + valid_candidate_val, + "duplicate case IDs", + ), + ( + valid_train, + _eval("candidate", "train", [("other", 1.0)]), + valid_val, + valid_candidate_val, + "case ID set mismatch", + ), + ( + _empty_eval("baseline", "train"), + _empty_eval("candidate", "train"), + valid_val, + valid_candidate_val, + "empty case results", + ), + ] + + for baseline_train, candidate_train, baseline_val, candidate_val, expected in scenarios: + decision = AcceptanceGate({ + "max_total_cost": None + }).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=[], + cost_summary=CostSummary(), + ) + assert decision.accepted is False + assert any(expected in reason for reason in decision.reasons) + assert decision.gate_status == "applied" + assert decision.not_applied_checks == [] + + +def test_case_delta_types_are_classified(): + baseline_train = _eval("baseline", "train", [("new_pass", 0.0), ("new_fail", 1.0)]) + candidate_train = _eval("candidate", "train", [("new_pass", 1.0), ("new_fail", 0.0)]) + baseline_val = _eval("baseline", "validation", [("up", 0.2), ("down", 0.8), ("same", 0.5)]) + candidate_val = _eval("candidate", "validation", [("up", 0.5), ("down", 0.5), ("same", 0.5)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + by_case = {delta.case_id: delta.delta_type for delta in deltas} + assert by_case["new_pass"] == "new_pass" + assert by_case["new_fail"] == "new_fail" + assert by_case["up"] == "score_up" + assert by_case["down"] == "score_down" + assert by_case["same"] == "unchanged" + + +def _eval(prompt_id: str, split: str, scores: list[tuple[str, float]]) -> EvalResult: + cases = [ + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="", + hard_failed=score <= 0.0, + ) for case_id, score in scores + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=round(sum(case.score for case in cases) / len(cases), 6), + passed=all(case.passed for case in cases), + cost=0.001 * len(cases), + cases=cases, + ) + + +def _empty_eval(prompt_id: str, split: str) -> EvalResult: + return EvalResult( + prompt_id=prompt_id, + split=split, + score=0.0, + passed=False, + cost=0.0, + cases=[], + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py new file mode 100644 index 00000000..a005bde1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DATA_FILES = [ + ROOT / "data" / "train.evalset.json", + ROOT / "data" / "val.evalset.json", +] +NO_SAMPLE_ID_FILES = [ + ROOT / "eval_loop" / "fake_model.py", + ROOT / "eval_loop" / "optimizer.py", + ROOT / "eval_loop" / "backends.py", +] +FORBIDDEN_MODEL_ACCESSES = [ + ".expectation", + ".split", + ".protected", + ".tags", + ".simulated_outputs", + ".case_id", +] + + +def test_sample_case_ids_do_not_appear_in_fake_runtime_logic(): + case_ids = { + case["eval_id"] + for path in DATA_FILES + for case in json.loads(path.read_text(encoding="utf-8"))["eval_cases"] + } + + for path in NO_SAMPLE_ID_FILES: + source = path.read_text(encoding="utf-8") + leaked = sorted(case_id for case_id in case_ids if case_id in source) + assert leaked == [], f"{path.name} contains sample case ids: {leaked}" + + +def test_fake_model_source_cannot_read_evaluator_only_case_fields(): + source = (ROOT / "eval_loop" / "fake_model.py").read_text(encoding="utf-8") + + leaked = [token for token in FORBIDDEN_MODEL_ACCESSES if token in source] + + assert leaked == [] + + +def test_fake_runtime_uses_no_private_optimizer_markers(): + for relative in ("eval_loop/fake_model.py", "eval_loop/optimizer.py"): + source = (ROOT / relative).read_text(encoding="utf-8") + assert "OPTIMIZER_MARKER" not in source diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py new file mode 100644 index 00000000..5ed95871 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline +from examples.optimization.eval_optimize_loop.eval_loop.report import report_to_json + + +def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): + output_dir = tmp_path / "run" + prompt_path = tmp_path / "baseline_system_prompt.txt" + prompt_path.write_text( + "You are a helpful support assistant.\n\n" + "Answer clearly and include a short explanation when it may help the user.\n" + "If the user asks for structured data, provide the information they need.\n", + encoding="utf-8", + ) + report = run_pipeline( + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=prompt_path, + output_dir=output_dir, + fake_model=True, + fake_judge=True, + trace=True, + ) + + json_path = output_dir / "optimization_report.json" + md_path = output_dir / "optimization_report.md" + assert json_path.is_file() + assert md_path.is_file() + assert report.selected_candidate == "candidate_002_safe" + assert report.schema_version == "eval_optimize_loop.v2" + + payload = json.loads(json_path.read_text(encoding="utf-8")) + assert payload["schema_version"] == "eval_optimize_loop.v2" + assert set(payload) >= { + "schema_version", + "run", + "baseline_train", + "baseline_validation", + "baseline", + "candidates", + "per_case_deltas", + "delta", + "failure_attribution_summary", + "gate_decisions", + "selected_candidate", + "audit", + } + assert payload["baseline_train"]["score"] == 0.333333 + assert payload["baseline_validation"]["score"] == 0.666667 + assert payload["selected_candidate"] == "candidate_002_safe" + assert [record["candidate"]["candidate_id"] for record in payload["candidates"]] == [ + "candidate_001_overfit", + "candidate_002_safe", + ] + assert len(payload["per_case_deltas"]) == 12 + assert payload["failure_attribution_summary"]["by_category"]["format_violation"] >= 1 + assert payload["failure_attribution_summary"]["attribution_accuracy"] == 1.0 + assert set(payload["audit"]) >= { + "seed", + "config_hash", + "cost", + "duration_seconds", + "prompt_hash", + "candidate_prompts", + "prompt_diffs", + "input_hashes", + "candidate_prompt_hashes", + "total_run_cost", + } + assert payload["audit"]["seed"] == 91 + assert payload["audit"]["duration_seconds"] > 0 + assert payload["run"]["trace"] is True + assert payload["run"]["reproducibility_command"].startswith("python examples/optimization") + assert payload["run"]["run_id"].startswith("eval_optimize_loop_fake_") + assert payload["audit"]["total_run_cost"] == payload["audit"]["cost"]["total"] + assert "candidate_001_overfit" in payload["audit"]["prompt_diffs"] + assert "candidate_002_safe" in payload["audit"]["prompt_diffs"] + assert payload["candidates"][0]["candidate"]["prompt_diff"].startswith("--- baseline_system_prompt.txt") + assert all(len(record["train_result"]["cases"]) == 3 for record in payload["candidates"]) + assert all(len(record["validation_result"]["cases"]) == 3 for record in payload["candidates"]) + assert payload["writeback"]["status"] == "not_requested" + + decisions = {item["candidate_id"]: item for item in payload["gate_decisions"]} + assert not decisions["candidate_001_overfit"]["accepted"] + assert decisions["candidate_002_safe"]["accepted"] + assert decisions["candidate_001_overfit"]["total_run_cost"] > decisions["candidate_001_overfit"]["candidate_cost"] + assert any("train score improved but validation score regressed" in reason + for reason in decisions["candidate_001_overfit"]["reasons"]) + + markdown = md_path.read_text(encoding="utf-8") + assert "Selected candidate: `candidate_002_safe`." in markdown + assert "candidate_001_overfit (rejected)" in markdown + assert "candidate_002_safe (accepted)" in markdown + assert "Baseline vs Candidate Scores" in markdown + assert "Per-Case Delta" in markdown + assert "Failure Attribution Summary" in markdown + assert "Prompt Diff" in markdown + assert "Reproducibility" in markdown + assert "Cost And Audit" in markdown + + run_dir = output_dir / "runs" / payload["run"]["run_id"] + overfit_artifact = payload["audit"]["candidate_artifacts"]["candidate_001_overfit"] + safe_artifact = payload["audit"]["candidate_artifacts"]["candidate_002_safe"] + assert (run_dir / "config.snapshot.json").is_file() + assert (run_dir / "input_hashes.json").is_file() + assert (run_dir / "candidate_prompts" / overfit_artifact / "system_prompt.txt").is_file() + assert (run_dir / "case_results" / f"{safe_artifact}_validation.json").is_file() + assert (run_dir / "prompt_diffs" / f"{safe_artifact}.diff").is_file() + assert (run_dir / "prompt_diffs" / f"{safe_artifact}.diff").read_text( + encoding="utf-8") == (payload["candidates"][1]["candidate"]["prompt_diff"]) + + +def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + run_pipeline(output_dir=first_dir, fake_model=True, fake_judge=True, trace=True) + run_pipeline(output_dir=second_dir, fake_model=True, fake_judge=True, trace=True) + + first_payload = json.loads((first_dir / "optimization_report.json").read_text(encoding="utf-8")) + second_payload = json.loads((second_dir / "optimization_report.json").read_text(encoding="utf-8")) + first_run_id = first_payload["run"]["run_id"] + second_run_id = second_payload["run"]["run_id"] + + assert _normalized_payload(first_payload) == _normalized_payload(second_payload) + first_markdown = (first_dir / "optimization_report.md").read_text(encoding="utf-8") + second_markdown = (second_dir / "optimization_report.md").read_text(encoding="utf-8") + normalized_first_markdown = first_markdown.replace(first_run_id, "").replace( + str(first_dir), + "", + ) + normalized_second_markdown = second_markdown.replace(second_run_id, "").replace( + str(second_dir), + "", + ) + assert normalized_first_markdown == normalized_second_markdown + + +def test_pipeline_accepts_mode_fake_without_legacy_flags(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + assert report.run["mode"] == "fake" + assert report.selected_candidate == "candidate_002_safe" + + +def test_pipeline_selected_candidate_is_null_when_all_candidates_rejected(tmp_path: Path): + config_path = tmp_path / "optimizer.json" + config = json.loads(Path(DEFAULT_OPTIMIZER_CONFIG).read_text(encoding="utf-8")) + config["gate"] = { + "allow_new_hard_fail": False, + "protected_case_ids": ["val_protected_yes_no"], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, + } + config["gate"]["min_val_score_improvement"] = 1.0 + config_path.write_text(json.dumps(config), encoding="utf-8") + + report = run_pipeline( + optimizer_config_path=config_path, + output_dir=tmp_path / "run", + mode="fake", + trace=True, + ) + + assert report.selected_candidate is None + + +def test_cli_mode_fake_and_legacy_fake_flags_both_run(tmp_path: Path): + script = Path("examples/optimization/eval_optimize_loop/run_pipeline.py") + first = tmp_path / "first" + second = tmp_path / "second" + + subprocess.run( + [sys.executable, str(script), "--mode", "fake", "--trace", "--output-dir", + str(first)], + check=True, + ) + subprocess.run( + [sys.executable, + str(script), "--fake-model", "--fake-judge", "--trace", "--output-dir", + str(second)], + check=True, + ) + + assert (first / "optimization_report.json").is_file() + assert (second / "optimization_report.md").is_file() + + +def test_report_to_json_rejects_nan_values(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + report.audit["bad_float"] = float("nan") + + try: + report_to_json(report) + except ValueError as exc: + assert "Out of range float values are not JSON compliant" in str(exc) + else: + raise AssertionError("report_to_json should reject NaN") + + +def test_fake_report_json_remains_strict_json(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + payload = report_to_json(report) + + assert "NaN" not in payload + assert "Infinity" not in payload + assert json.loads(payload)["selected_candidate"] == "candidate_002_safe" + + +def _normalized_payload(payload: dict) -> dict: + normalized = json.loads(json.dumps(payload)) + normalized["run"].pop("run_id", None) + normalized["run"].pop("reproducibility_command", None) + normalized["audit"].pop("duration_seconds", None) + normalized["audit"].pop("reproducibility_command", None) + normalized["audit"].get("sdk_result_summary", {}).pop( + "proposal_duration_seconds", + None, + ) + normalized["audit"].get("writeback_journal", {}).pop("run_id", None) + for round_record in normalized.get("rounds", []): + round_record.pop("duration_seconds", None) + return normalized diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py new file mode 100644 index 00000000..14b56567 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py @@ -0,0 +1,2177 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import math +import os +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.eval_loop import report as report_module +from examples.optimization.eval_optimize_loop.eval_loop.backends import FakeBackend +from examples.optimization.eval_optimize_loop.eval_loop.pipeline import PipelineRequest +from examples.optimization.eval_optimize_loop.eval_loop.pipeline import execute_pipeline +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CandidatePrompt +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationRound +from examples.optimization.eval_optimize_loop.eval_loop.schemas import WritebackResult +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ConcurrentPromptUpdateError +from examples.optimization.eval_optimize_loop.eval_loop.writeback import PromptRestorationError +from examples.optimization.eval_optimize_loop.run_pipeline import build_pipeline_request_and_backend + + +class RecordingBackend: + + def __init__( + self, + *, + candidates: list[CandidatePrompt], + results: dict[tuple[str, str], EvalResult], + optimization_cost: CostSummary | None = None, + rounds: list[OptimizationRound] | None = None, + raw_summary: dict[str, object] | None = None, + ) -> None: + self.candidates = candidates + self.results = results + self.optimization_cost = optimization_cost or CostSummary() + self.rounds = list(rounds or []) + self.raw_summary = raw_summary or {"status": "SUCCEEDED"} + self.calls: list[tuple[Any, ...]] = [] + self.failure_summary: dict[str, object] | None = None + + async def evaluate( + self, + *, + prompt_id: str, + prompts: dict[str, str], + dataset_path: str | Path, + split: str, + trace: bool, + artifact_dir: str | Path, + ) -> EvalResult: + self.calls.append(("evaluate", prompt_id, split, dict(prompts), Path(artifact_dir), trace)) + return self.results[(prompt_id, split)] + + async def optimize_candidates( + self, + *, + baseline_prompts: dict[str, str], + baseline_train: EvalResult, + failure_summary: dict[str, object], + train_path: str | Path, + validation_path: str | Path, + config_path: str | Path, + artifact_dir: str | Path, + ) -> OptimizationResult: + self.calls.append(("optimize", dict(baseline_prompts), Path(artifact_dir))) + self.failure_summary = failure_summary + return OptimizationResult( + candidates=self.candidates, + rounds=self.rounds, + cost=self.optimization_cost, + raw_summary=self.raw_summary, + ) + + +class MutatingBundleBackend(RecordingBackend): + + async def evaluate(self, **kwargs: Any) -> EvalResult: + result = await super().evaluate(**kwargs) + kwargs["prompts"]["system_prompt"] = "backend-mutated prompt" + return result + + async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: + result = await super().optimize_candidates(**kwargs) + kwargs["baseline_prompts"]["system_prompt"] = "optimizer-mutated baseline" + return result + + +class FailingCandidateBackend(RecordingBackend): + + def __init__( + self, + *, + fail_on: tuple[str, str], + failure: BaseException, + delegate: RecordingBackend, + ) -> None: + super().__init__( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + self.fail_on = fail_on + self.failure = failure + + async def evaluate(self, **kwargs: Any) -> EvalResult: + if (kwargs["prompt_id"], kwargs["split"]) == self.fail_on: + self.calls.append(( + "evaluate", + kwargs["prompt_id"], + kwargs["split"], + dict(kwargs["prompts"]), + Path(kwargs["artifact_dir"]), + kwargs["trace"], + )) + raise self.failure + return await super().evaluate(**kwargs) + + +class AllCandidatesFailingBackend(RecordingBackend): + + async def evaluate(self, **kwargs: Any) -> EvalResult: + if kwargs["prompt_id"] != "baseline": + self.calls.append(( + "evaluate", + kwargs["prompt_id"], + kwargs["split"], + dict(kwargs["prompts"]), + Path(kwargs["artifact_dir"]), + kwargs["trace"], + )) + raise RuntimeError(f"{kwargs['prompt_id']} evaluation unavailable") + return await super().evaluate(**kwargs) + + +class SourceMutatingBackend(RecordingBackend): + + def __init__( + self, + *, + request: PipelineRequest, + mutate_on: tuple[str, str] | str, + failure: BaseException | None, + delegate: RecordingBackend, + ) -> None: + super().__init__( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + self.request = request + self.mutate_on = mutate_on + self.failure = failure + + def _mutate_source(self) -> None: + Path(self.request.target_prompt_paths["system_prompt"]).write_text( + "backend source mutation", + encoding="utf-8", + ) + + async def evaluate(self, **kwargs: Any) -> EvalResult: + result = await super().evaluate(**kwargs) + if (kwargs["prompt_id"], kwargs["split"]) == self.mutate_on: + self._mutate_source() + if self.failure is not None: + raise self.failure + return result + + async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: + result = await super().optimize_candidates(**kwargs) + if self.mutate_on == "optimizer": + self._mutate_source() + if self.failure is not None: + raise self.failure + return result + + +@pytest.mark.asyncio +async def test_execute_pipeline_uses_train_only_evidence_and_full_candidate_reevaluation(tmp_path: Path): + request = _request(tmp_path, update_source=False) + backend = _safe_backend() + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert [(call[0], *call[1:3]) for call in backend.calls] == [ + ("evaluate", "baseline", "train"), + ("evaluate", "baseline", "validation"), + ( + "optimize", + { + "system_prompt": "baseline\n" + }, + tmp_path / "out" / "runs" / ".ordered_run.tmp" / "optimizer", + ), + ("evaluate", "candidate_a", "train"), + ("evaluate", "candidate_a", "validation"), + ("evaluate", "candidate_b", "train"), + ("evaluate", "candidate_b", "validation"), + ] + assert backend.failure_summary is not None + assert backend.failure_summary["total_failed_cases"] == 1 + assert backend.failure_summary["by_category"] == {"train_only": 1} + assert "validation_only" not in str(backend.failure_summary) + assert report.selected_candidate == "candidate_a" + assert report.audit["duration_seconds"] > 0 + assert report.audit["config_snapshot"] == {"seed": 91} + assert report.audit["input_hashes"]["target_prompts"]["system_prompt"] == hashlib.sha256(b"baseline\n").hexdigest() + artifact_dirs = [call[4] for call in backend.calls if call[0] == "evaluate"] + assert len(artifact_dirs) == len(set(artifact_dirs)) == 6 + assert report.audit["sdk_result_availability"]["full_train_eval_result"] is True + assert report.audit["sdk_result_availability"]["full_per_case_validation_delta"] is True + + +@pytest.mark.asyncio +async def test_backend_cannot_mutate_canonical_baseline_or_candidate_bundles(tmp_path: Path): + request = _request(tmp_path, update_source=True) + delegate = _safe_backend(candidate_count=1) + backend = MutatingBundleBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + prompt_calls = [(call[1], call[2], call[3]["system_prompt"]) for call in backend.calls if call[0] == "evaluate"] + assert prompt_calls == [ + ("baseline", "train", "baseline\n"), + ("baseline", "validation", "baseline\n"), + ("candidate_a", "train", "safe prompt"), + ("candidate_a", "validation", "safe prompt"), + ] + optimize_call = next(call for call in backend.calls if call[0] == "optimize") + assert optimize_call[1] == {"system_prompt": "baseline\n"} + assert report.candidates[0]["candidate"].bundle() == {"system_prompt": "safe prompt"} + assert report.candidates[0]["prompt_bundle"] == {"system_prompt": "safe prompt"} + assert report.audit["candidate_prompts"]["candidate_a"] == {"system_prompt": "safe prompt"} + assert report.audit["candidate_prompt_hashes"]["candidate_a"] == { + "system_prompt": hashlib.sha256(b"safe prompt").hexdigest() + } + candidate_artifact = report.audit["candidate_artifacts"]["candidate_a"] + artifact_prompt = Path(request.output_dir).joinpath( + "runs", + request.run_id, + "candidate_prompts", + candidate_artifact, + "system_prompt.txt", + ) + assert artifact_prompt.read_text(encoding="utf-8") == "safe prompt" + assert Path(request.target_prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "safe prompt" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("mutate_on", "update_source"), + [ + (("baseline", "train"), False), + (("baseline", "validation"), False), + ("optimizer", False), + (("candidate_a", "train"), False), + (("candidate_a", "validation"), True), + ], +) +async def test_successful_backend_source_mutation_is_fatal( + tmp_path: Path, + mutate_on: tuple[str, str] | str, + update_source: bool, +): + request = _request(tmp_path, update_source=update_source) + backend = SourceMutatingBackend( + request=request, + mutate_on=mutate_on, + failure=None, + delegate=_safe_backend(candidate_count=1), + ) + + with pytest.raises(ConcurrentPromptUpdateError, match="source prompt.*changed"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert Path(request.target_prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "backend source mutation" + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure", + [RuntimeError("backend error after mutation"), + asyncio.CancelledError()], + ids=["error", "cancellation"], +) +async def test_backend_error_or_cancellation_with_source_mutation_is_integrity_fatal( + tmp_path: Path, + failure: BaseException, +): + request = _request(tmp_path, update_source=False) + backend = SourceMutatingBackend( + request=request, + mutate_on=("candidate_a", "train"), + failure=failure, + delegate=_safe_backend(candidate_count=2), + ) + + with pytest.raises( + ConcurrentPromptUpdateError, + match="source prompt.*changed", + ) as error_info: + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert error_info.value.__cause__ is failure + assert not any(call[0] == "evaluate" and call[1] == "candidate_b" for call in backend.calls) + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +async def test_clean_backend_cancellation_preserves_cancelled_error(tmp_path: Path): + request = _request(tmp_path, update_source=False) + cancellation = asyncio.CancelledError() + backend = FailingCandidateBackend( + fail_on=("candidate_a", "train"), + failure=cancellation, + delegate=_safe_backend(candidate_count=2), + ) + + with pytest.raises(asyncio.CancelledError) as error_info: + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert error_info.value is cancellation + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + assert not any(call[0] == "evaluate" and call[1] == "candidate_b" for call in backend.calls) + + +@pytest.mark.asyncio +async def test_unreadable_source_prompt_is_integrity_fatal_without_path_leak(tmp_path: Path): + request = _request(tmp_path, update_source=False) + backend = SourceMutatingBackend( + request=request, + mutate_on=("baseline", "train"), + failure=None, + delegate=_safe_backend(candidate_count=1), + ) + + def remove_source() -> None: + Path(request.target_prompt_paths["system_prompt"]).unlink() + + backend._mutate_source = remove_source # type: ignore[method-assign] + + with pytest.raises(PromptRestorationError, match="source prompt field") as error_info: + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert str(tmp_path) not in str(error_info.value) + assert isinstance(error_info.value.__cause__, OSError) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_split", ["train", "validation"]) +async def test_candidate_evaluation_error_rejects_candidate_and_continues( + tmp_path: Path, + failed_split: str, +): + request = _request(tmp_path, update_source=True) + backend = FailingCandidateBackend( + fail_on=("candidate_a", failed_split), + failure=RuntimeError(f"synthetic {failed_split} evaluation failure"), + delegate=_safe_backend(candidate_count=2), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + candidate_calls = [(call[1], call[2]) for call in backend.calls if call[0] == "evaluate" and call[1] != "baseline"] + expected_a_calls = [("candidate_a", "train")] + if failed_split == "validation": + expected_a_calls.append(("candidate_a", "validation")) + assert candidate_calls == expected_a_calls + [ + ("candidate_b", "train"), + ("candidate_b", "validation"), + ] + assert report.selected_candidate == "candidate_b" + assert report.writeback.status == "applied" + assert Path(request.target_prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "other prompt" + + failed_record = report.candidates[0] + assert failed_record["candidate"].candidate_id == "candidate_a" + assert failed_record["evaluation_error"]["stage"] == failed_split + assert failed_record["evaluation_error"]["type"] == "RuntimeError" + assert failed_record["evaluation_error"]["cost_complete"] is False + assert ("train_result" in failed_record) is (failed_split == "validation") + assert "validation_result" not in failed_record + + failed_decision = report.gate_decisions[0] + assert failed_decision.accepted is False + assert failed_decision.gate_status == "not_applied" + assert failed_decision.gate_not_applied_reason == "evaluation_error" + assert failed_decision.train_score_delta is None + assert failed_decision.validation_score_delta is None + assert any("evaluation_error" in reason and failed_split in reason for reason in failed_decision.reasons) + assert failed_decision.not_applied_checks + + audit_failure = report.audit["candidate_evaluation_failures"]["candidate_a"] + assert audit_failure == failed_record["evaluation_error"] + assert report.cost_summary.complete is False + expected_known_cost = 0.05 if failed_split == "validation" else 0.04 + assert report.cost_summary.total == expected_known_cost + assert report.audit["cost"]["total"] is None + assert report.audit["cost"]["known_run_cost"] == expected_known_cost + markdown = report_module.render_markdown(report) + assert "evaluation_error" in markdown + assert "n/a" in markdown + + +@pytest.mark.asyncio +async def test_candidate_evaluation_error_makes_later_budget_gate_fail_closed(tmp_path: Path, ): + request = _request( + tmp_path, + update_source=True, + gate_config={ + "min_val_score_improvement": 0.0, + "max_score_drop_per_case": 1.0, + "max_total_cost": 10.0, + }, + ) + backend = FailingCandidateBackend( + fail_on=("candidate_a", "train"), + failure=RuntimeError("unknown failed evaluation cost"), + delegate=_safe_backend(candidate_count=2), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.writeback.status == "rejected" + assert report.gate_decisions[0].gate_not_applied_reason == "evaluation_error" + assert any("cost_unavailable" in reason for reason in report.gate_decisions[1].reasons) + assert report.cost_summary.complete is False + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + + +@pytest.mark.asyncio +async def test_all_candidate_evaluation_errors_finalize_rejected_audit_without_writeback(tmp_path: Path, ): + request = _request(tmp_path, update_source=True) + delegate = _safe_backend(candidate_count=2) + backend = AllCandidatesFailingBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.writeback.status == "rejected" + assert report.audit["writeback_journal"]["state"] == "rejected" + assert report.audit["writeback_journal"]["report_phase"] == "final" + assert set(report.audit["candidate_evaluation_failures"]) == { + "candidate_a", + "candidate_b", + } + assert all("evaluation_error" in record for record in report.candidates) + assert all("train_result" not in record for record in report.candidates) + assert all(decision.gate_status == "not_applied" for decision in report.gate_decisions) + assert report.cost_summary.complete is False + assert report.cost_summary.total == 0.02 + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + run_dir = Path(request.output_dir) / "runs" / request.run_id + assert (run_dir / "optimization_report.json").is_file() + assert (run_dir / "audit.json").is_file() + assert (run_dir / "writeback.json").is_file() + + +@pytest.mark.asyncio +async def test_candidate_evaluation_error_does_not_persist_backend_secret_text(tmp_path: Path, ): + request = _request(tmp_path) + secret = "api_key=sk-private-value token=github-private-value" + backend = FailingCandidateBackend( + fail_on=("candidate_a", "train"), + failure=RuntimeError(secret), + delegate=_safe_backend(candidate_count=2), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + failure = report.candidates[0]["evaluation_error"] + assert failure["message"] == "candidate evaluation failed; backend details withheld" + assert failure["message_sha256"] == hashlib.sha256(secret.encode("utf-8")).hexdigest() + serialized = report_module.report_to_json(report) + report_module.render_markdown(report) + serialized += (Path(request.output_dir) / "runs" / request.run_id / "audit.json").read_text(encoding="utf-8") + assert "sk-private-value" not in serialized + assert "github-private-value" not in serialized + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_kind", ["cas", "direct_restore", "chained_restore"]) +async def test_candidate_prompt_integrity_failures_abort_run( + tmp_path: Path, + failure_kind: str, +): + request = _request(tmp_path, update_source=True) + if failure_kind == "cas": + failure: Exception = ConcurrentPromptUpdateError("source changed") + elif failure_kind == "direct_restore": + failure = PromptRestorationError("restore failed") + else: + failure = RuntimeError("candidate evaluation failed") + failure.__cause__ = PromptRestorationError("restore failed") + backend = FailingCandidateBackend( + fail_on=("candidate_a", "train"), + failure=failure, + delegate=_safe_backend(candidate_count=2), + ) + + with pytest.raises(Exception) as error_info: + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert error_info.value is failure + assert not any(call[0] == "evaluate" and call[1] == "candidate_b" for call in backend.calls) + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("mutation_target", "error_pattern"), + [ + ("prompt", "source prompt integrity changed"), + ("input", "immutable input snapshot.*train"), + ], +) +async def test_candidate_failure_with_state_drift_aborts_run( + tmp_path: Path, + mutation_target: str, + error_pattern: str, +): + request = _request(tmp_path, update_source=True) + + class StateDriftBackend(FailingCandidateBackend): + + async def evaluate(self, **kwargs: Any) -> EvalResult: + if (kwargs["prompt_id"], kwargs["split"]) == self.fail_on: + if mutation_target == "prompt": + Path(request.target_prompt_paths["system_prompt"]).write_text( + "external prompt update", + encoding="utf-8", + ) + else: + Path(kwargs["dataset_path"]).write_text("{}", encoding="utf-8") + return await super().evaluate(**kwargs) + + backend = StateDriftBackend( + fail_on=("candidate_a", "train"), + failure=RuntimeError("ordinary evaluation failure"), + delegate=_safe_backend(candidate_count=2), + ) + + with pytest.raises((ConcurrentPromptUpdateError, ValueError), match=error_pattern): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert not any(call[0] == "evaluate" and call[1] == "candidate_b" for call in backend.calls) + + +@pytest.mark.asyncio +async def test_rejected_candidate_never_changes_source_when_writeback_requested(tmp_path: Path): + request = _request( + tmp_path, + update_source=True, + gate_config={ + "min_val_score_improvement": 0.5, + "max_total_cost": None + }, + ) + backend = _safe_backend(candidate_validation_score=0.0) + before = (tmp_path / "prompt.txt").read_bytes() + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.writeback.status == "rejected" + assert report.writeback.before_hashes == { + "system_prompt": hashlib.sha256(before).hexdigest(), + } + assert report.audit["writeback_journal"]["report_phase"] == "final" + assert (tmp_path / "prompt.txt").read_bytes() == before + + +@pytest.mark.asyncio +async def test_accepted_candidate_is_not_written_when_writeback_not_requested(tmp_path: Path): + request = _request(tmp_path, update_source=False) + backend = _safe_backend() + before = (tmp_path / "prompt.txt").read_bytes() + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate == "candidate_a" + assert report.writeback.status == "not_requested" + assert report.writeback.before_hashes == { + "system_prompt": hashlib.sha256(before).hexdigest(), + } + assert (tmp_path / "prompt.txt").read_bytes() == before + + +@pytest.mark.asyncio +async def test_writeback_happens_only_after_prewrite_audit_succeeds( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend() + events: list[str] = [] + artifact_paths: report_module.RunArtifactPaths | None = None + + def prepare(report, paths): + nonlocal artifact_paths + events.append("prepare") + assert report.selected_candidate == "candidate_a" + assert paths.output_dir == tmp_path / "out" + assert paths.temp_run_dir == tmp_path / "out" / "runs" / ".ordered_run.tmp" + artifact_paths = paths + return paths + + def commit(snapshot, prompts): + events.append("commit") + assert prompts == {"system_prompt": "safe prompt"} + return WritebackResult( + status="applied", + before_hashes=snapshot.hashes(), + after_hashes=snapshot.hashes(), + ) + + def finalize(report, paths, *, before_publish): + events.append("finalize") + assert paths is artifact_paths + assert report.writeback.status == "applied" + before_publish() + + def persist_journal(paths, journal): + events.append(f"journal:{journal['state']}") + assert paths is artifact_paths + + def persist_outcome(report, paths): + events.append(f"outcome:{report.audit['writeback_journal']['state']}") + assert paths is artifact_paths + + monkeypatch.setattr(pipeline_module, "prepare_run_artifacts", prepare) + monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", commit) + monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", finalize) + monkeypatch.setattr(pipeline_module, "persist_writeback_journal", persist_journal) + monkeypatch.setattr(pipeline_module, "persist_writeback_outcome", persist_outcome) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert events == [ + "prepare", + "journal:committing", + "commit", + "journal:applied", + "outcome:applied", + "finalize", + ] + assert report.writeback.status == "applied" + + +@pytest.mark.asyncio +async def test_prepare_failure_prevents_source_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend() + before = (tmp_path / "prompt.txt").read_bytes() + commit_called = False + + def fail_prepare(report, output_dir): + raise OSError("audit unavailable") + + def forbidden_commit(snapshot, prompts): + nonlocal commit_called + commit_called = True + raise AssertionError("commit must not run") + + monkeypatch.setattr(pipeline_module, "prepare_run_artifacts", fail_prepare) + monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", forbidden_commit) + + with pytest.raises(OSError, match="audit unavailable"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert commit_called is False + assert (tmp_path / "prompt.txt").read_bytes() == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("max_total_cost", "accepted", "reason_fragment"), + [ + (1.0, False, "cost_unavailable"), + (None, True, None), + ], +) +async def test_incomplete_optimizer_cost_is_fail_closed_only_when_budget_configured( + tmp_path: Path, + max_total_cost: float | None, + accepted: bool, + reason_fragment: str | None, +): + request = _request( + tmp_path, + gate_config={ + "min_val_score_improvement": 0.0, + "max_score_drop_per_case": 1.0, + "max_total_cost": max_total_cost, + }, + ) + backend = _safe_backend( + candidate_count=1, + optimization_cost=CostSummary(optimizer=0.2, total=0.2, complete=False), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.gate_decisions[0].accepted is accepted + if reason_fragment is not None: + assert any(reason_fragment in reason for reason in report.gate_decisions[0].reasons) + else: + assert all("cost_unavailable" not in reason for reason in report.gate_decisions[0].reasons) + payload = json.loads(report_module.report_to_json(report)) + assert payload["cost_summary"]["complete"] is False + assert payload["cost_summary"]["reported_optimizer_cost"] == 0.2 + assert report.audit["cost"]["complete"] is False + assert report.audit["cost"]["total"] is None + assert report.audit["cost"]["reported_optimizer_cost"] == 0.2 + markdown = report_module.render_markdown(report) + assert "Reported optimizer cost (incomplete; not total run cost): 0.200" in markdown + assert "Total cost:" not in markdown + + +@pytest.mark.asyncio +async def test_optimizer_cost_is_counted_once_across_candidates(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend( + baseline_train_cost=0.1, + baseline_validation_cost=0.2, + candidate_train_cost=0.04, + candidate_validation_cost=0.05, + optimization_cost=CostSummary(optimizer=0.3, total=0.3, complete=True), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert [decision.total_run_cost for decision in report.gate_decisions] == [0.69, 0.78] + assert report.cost_summary == CostSummary( + optimizer=0.3, + evaluator=0.48, + agent=0.0, + total=0.78, + complete=True, + ) + assert report.audit["total_run_cost"] == 0.78 + assert report.audit["known_run_cost"] is None + assert report.audit["total_run_cost_complete"] is True + assert report.audit["cost"]["total"] == 0.78 + assert report.audit["cost"]["reported_optimizer_cost"] is None + markdown = report_module.render_markdown(report) + assert "Total cost: 0.780" in markdown + assert "Reported optimizer cost" not in markdown + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("baseline_case_ids", "candidate_case_ids", "reason_fragment"), + [ + (["a"], ["a", "a"], "duplicate case IDs"), + (["a", "b"], ["a"], "case ID set mismatch"), + (["a"], ["a", "extra"], "case ID set mismatch"), + ], +) +async def test_incomparable_case_results_are_rejected_without_delta_error( + tmp_path: Path, + baseline_case_ids: list[str], + candidate_case_ids: list[str], + reason_fragment: str, +): + request = _request(tmp_path) + for dataset_path in (Path(request.train_path), Path(request.validation_path)): + dataset_path.write_text( + json.dumps({ + "eval_cases": [{ + "eval_id": case_id, + "session_input": { + "state": {} + } + } for case_id in dict.fromkeys(baseline_case_ids)] + }), + encoding="utf-8", + ) + candidate = CandidatePrompt("candidate_a", "safe prompt", "safe", "diff") + results = { + ("baseline", "train"): _eval("baseline", "train", baseline_case_ids, 0.0), + ("baseline", "validation"): _eval("baseline", "validation", baseline_case_ids, 0.0), + ("candidate_a", "train"): _eval("candidate_a", "train", candidate_case_ids, 1.0), + ("candidate_a", "validation"): _eval("candidate_a", "validation", candidate_case_ids, 1.0), + } + backend = RecordingBackend(candidates=[candidate], results=results) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.per_case_deltas == [] + assert report.gate_decisions[0].accepted is False + assert any(reason_fragment in reason for reason in report.gate_decisions[0].reasons) + + +@pytest.mark.asyncio +async def test_empty_evalsets_are_rejected_before_backend_calls(tmp_path: Path): + request = _request(tmp_path) + Path(request.train_path).write_text('{"eval_cases": []}', encoding="utf-8") + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="train evalset.*must not be empty"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + + +@pytest.mark.asyncio +async def test_duplicate_candidate_ids_are_rejected(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + backend.candidates = [backend.candidates[0], backend.candidates[0]] + + with pytest.raises(ValueError, match="duplicate candidate_id"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("prompt_fields", "message"), + [ + ({ + "other_prompt": "candidate" + }, "bundle fields"), + ({ + "system_prompt": "" + }, "non-empty string"), + ], +) +async def test_candidate_bundle_must_match_target_snapshot( + tmp_path: Path, + prompt_fields: dict[str, str], + message: str, +): + request = _request(tmp_path) + candidate = CandidatePrompt( + candidate_id="candidate_a", + prompt="candidate", + rationale="test", + prompt_diff="diff", + prompt_fields=prompt_fields, + ) + backend = _safe_backend(candidate_count=0) + backend.candidates = [candidate] + + with pytest.raises(ValueError, match=message): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_target_prompt_hashes_cannot_overwrite_dataset_hashes(tmp_path: Path): + request = _request(tmp_path) + prompt_path = tmp_path / "reserved_name_prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + request = replace(request, target_prompt_paths={"train": prompt_path}) + backend = _safe_backend(candidate_count=1) + backend.candidates = [ + CandidatePrompt( + candidate_id="candidate_a", + prompt="safe prompt", + rationale="safe", + prompt_diff="diff", + prompt_fields={"train": "safe prompt"}, + ) + ] + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + expected_dataset_hash = hashlib.sha256(Path(request.train_path).read_bytes()).hexdigest() + expected_prompt_hash = hashlib.sha256(prompt_path.read_bytes()).hexdigest() + assert report.audit["input_hashes"]["train"] == expected_dataset_hash + assert report.audit["input_hashes"]["target_prompts"]["train"] == expected_prompt_hash + + +@pytest.mark.asyncio +async def test_finalize_failure_leaves_explicit_prepared_writeback_journal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + + def fail_finalize(report, paths, *, before_publish): + raise OSError("final report unavailable") + + monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", fail_finalize) + + with pytest.raises(OSError, match="final report unavailable"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + run_dir = Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" + prewrite = json.loads((run_dir / "pre_write_report.json").read_text(encoding="utf-8")) + journal = json.loads((run_dir / "writeback_journal.json").read_text(encoding="utf-8")) + assert prewrite["audit"]["writeback_journal"]["state"] == "pending" + assert "pending" in prewrite["writeback"]["error"] + assert journal["state"] == "applied" + assert (tmp_path / "prompt.txt").read_bytes() == b"safe prompt" + assert journal["after_hashes"] == { + "system_prompt": hashlib.sha256(b"safe prompt").hexdigest(), + } + assert not (Path(request.output_dir) / "runs" / request.run_id).exists() + + +@pytest.mark.asyncio +async def test_writeback_conflict_is_caught_and_durably_reported( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + + def conflict(snapshot, prompts): + raise ConcurrentPromptUpdateError("source changed") + + monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", conflict) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert report.writeback.status == "rejected" + assert "source changed" in (report.writeback.error or "") + assert journal["state"] == "conflict" + assert journal["error"] == report.writeback.error + + +@pytest.mark.asyncio +async def test_real_source_drift_between_prepare_and_commit_is_terminal_conflict( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + original_prepare = pipeline_module.prepare_run_artifacts + + def prepare_then_external_change(report, output_dir): + paths = original_prepare(report, output_dir) + (tmp_path / "prompt.txt").write_bytes(b"external writer\n") + return paths + + monkeypatch.setattr(pipeline_module, "prepare_run_artifacts", prepare_then_external_change) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + journal_path = Path(request.output_dir) / "runs" / request.run_id / "writeback_journal.json" + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert report.writeback.status == "rejected" + assert report.writeback.after_hashes == { + "system_prompt": hashlib.sha256(b"external writer\n").hexdigest(), + } + assert journal["state"] == "conflict" + assert journal["after_hashes"] == report.writeback.after_hashes + assert (tmp_path / "prompt.txt").read_bytes() == b"external writer\n" + + +@pytest.mark.asyncio +async def test_terminal_journal_survives_nonessential_writeback_artifact_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + + def fail_outcome(report, paths): + raise OSError("writeback convenience artifact unavailable") + + monkeypatch.setattr(pipeline_module, "persist_writeback_outcome", fail_outcome) + + with pytest.raises(OSError, match="convenience artifact"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + journal_path = (Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "writeback_journal.json") + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert (tmp_path / "prompt.txt").read_bytes() == b"safe prompt" + assert journal["state"] == "applied" + assert journal["after_hashes"] == { + "system_prompt": hashlib.sha256(b"safe prompt").hexdigest(), + } + + +@pytest.mark.asyncio +async def test_writeback_journal_is_committing_before_source_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + real_commit = pipeline_module.commit_prompt_bundle + + def commit(snapshot, prompts): + journal_path = (Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "writeback_journal.json") + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert journal["state"] == "committing" + return real_commit(snapshot, prompts) + + monkeypatch.setattr(pipeline_module, "commit_prompt_bundle", commit) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.writeback.status == "applied" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("writeback_status", ["rolled_back", "rollback_failed"]) +async def test_failed_writeback_is_persisted_in_final_report_and_journal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + writeback_status: str, +): + request = _request(tmp_path, update_source=True) + backend = _safe_backend(candidate_count=1) + monkeypatch.setattr( + pipeline_module, + "commit_prompt_bundle", + lambda snapshot, prompts: WritebackResult( + status=writeback_status, + before_hashes=snapshot.hashes(), + after_hashes=snapshot.hashes(), + error="simulated commit failure", + ), + ) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + run_dir = Path(request.output_dir) / "runs" / request.run_id + final_payload = json.loads((run_dir / "optimization_report.json").read_text(encoding="utf-8")) + writeback_payload = json.loads((run_dir / "writeback.json").read_text(encoding="utf-8")) + journal = json.loads((run_dir / "writeback_journal.json").read_text(encoding="utf-8")) + assert report.writeback.status == writeback_status + assert final_payload["writeback"]["status"] == writeback_status + assert writeback_payload["status"] == writeback_status + assert journal["state"] == writeback_status + + +@pytest.mark.asyncio +async def test_standard_evalset_protected_metadata_is_applied_to_gate(tmp_path: Path): + request = _request(tmp_path) + Path(request.train_path).write_text( + json.dumps({"eval_cases": [{ + "eval_id": "train", + "session_input": { + "state": {} + } + }]}), + encoding="utf-8", + ) + Path(request.validation_path).write_text( + """{ + "eval_set_id": "validation", + "eval_cases": [ + { + "eval_id": "protected", + "session_input": {"state": {"eval_optimize_protected": true}} + }, + { + "eval_id": "improved", + "session_input": {"state": {"eval_optimize_protected": false}} + } + ] +} +""", + encoding="utf-8", + ) + candidate = CandidatePrompt("candidate_a", "safe prompt", "safe", "diff") + results = { + ("baseline", "train"): _eval_scores("baseline", "train", [("train", 1.0)]), + ("baseline", "validation"): _eval_scores( + "baseline", + "validation", + [("protected", 0.8), ("improved", 0.0)], + ), + ("candidate_a", "train"): _eval_scores("candidate_a", "train", [("train", 1.0)]), + ("candidate_a", "validation"): _eval_scores( + "candidate_a", + "validation", + [("protected", 0.7), ("improved", 1.0)], + ), + } + backend = RecordingBackend(candidates=[candidate], results=results) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.selected_candidate is None + assert report.gate_decisions[0].protected_regressions == ["protected"] + assert report.audit["gate_config_snapshot"]["protected_case_ids"] == ["protected"] + + +@pytest.mark.asyncio +async def test_baseline_results_must_cover_every_snapshotted_dataset_case(tmp_path: Path): + request = _request(tmp_path, update_source=True) + Path(request.validation_path).write_text( + json.dumps({ + "eval_cases": [ + { + "eval_id": "validation_case", + "session_input": { + "state": {} + } + }, + { + "eval_id": "protected_missing", + "session_input": { + "state": { + "eval_optimize_protected": True + } + }, + }, + ] + }), + encoding="utf-8", + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="baseline.*validation.*dataset case IDs"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert (tmp_path / "prompt.txt").read_bytes() == b"baseline\n" + + +@pytest.mark.asyncio +async def test_backend_neutral_core_rejects_same_resolved_train_and_validation_path(tmp_path: Path): + request = _request(tmp_path) + request = replace(request, validation_path=request.train_path) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="train and validation.*different"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + + +@pytest.mark.asyncio +async def test_backend_neutral_core_rejects_hardlinked_train_and_validation(tmp_path: Path, ): + request = _request(tmp_path) + validation_path = Path(request.validation_path) + validation_path.unlink() + os.link(request.train_path, validation_path) + assert Path(request.train_path).samefile(validation_path) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="train and validation.*different physical files"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + assert not (Path(request.output_dir) / "runs").exists() + + +def test_factory_rejects_hardlinked_train_and_validation_before_backend_creation(tmp_path: Path, ): + request = _request(tmp_path) + validation_path = Path(request.validation_path) + validation_path.unlink() + os.link(request.train_path, validation_path) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="train and validation.*different physical files"): + build_pipeline_request_and_backend( + train_path=request.train_path, + val_path=validation_path, + optimizer_config_path=request.optimizer_config_path, + prompt_path=request.target_prompt_paths["system_prompt"], + output_dir=request.output_dir, + mode="fake", + backend=backend, + ) + + assert backend.calls == [] + assert not Path(request.output_dir).exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field_names", [["Foo", "foo"], ["CON"]]) +async def test_backend_neutral_core_rejects_unsafe_or_colliding_target_fields( + tmp_path: Path, + field_names: list[str], +): + request = _request(tmp_path) + target_paths: dict[str, Path] = {} + for index, field_name in enumerate(field_names): + prompt_path = tmp_path / f"prompt-{index}.txt" + prompt_path.write_text(f"prompt {index}", encoding="utf-8") + target_paths[field_name] = prompt_path + request = replace(request, target_prompt_paths=target_paths) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="target prompt field.*unsafe|case-insensitively unique"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + + +@pytest.mark.asyncio +async def test_backend_neutral_core_rejects_target_fields_for_same_resolved_file(tmp_path: Path, ): + request = _request(tmp_path) + prompt_path = request.target_prompt_paths["system_prompt"] + request = replace( + request, + target_prompt_paths={ + "system_prompt": prompt_path, + "router_prompt": prompt_path + }, + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="target prompt fields.*different physical files"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("alias_kind", ["hardlink", "casefold"]) +async def test_backend_neutral_core_rejects_physical_or_casefold_target_aliases( + tmp_path: Path, + alias_kind: str, +): + request = _request(tmp_path) + if alias_kind == "hardlink": + first_path = Path(request.target_prompt_paths["system_prompt"]) + second_path = tmp_path / "prompt-hardlink.txt" + os.link(first_path, second_path) + assert first_path.samefile(second_path) + else: + first_path = tmp_path / "CasePrompt.txt" + second_path = tmp_path / "caseprompt.txt" + first_path.write_text("first", encoding="utf-8") + second_path.write_text("second", encoding="utf-8") + request = replace( + request, + target_prompt_paths={ + "system_prompt": first_path, + "router_prompt": second_path + }, + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="target prompt fields.*different physical files"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert backend.calls == [] + assert not (Path(request.output_dir) / "runs").exists() + + +def test_shared_path_validator_handles_missing_casefold_collisions(tmp_path: Path): + pipeline_module.validate_distinct_file_paths( + { + "first": tmp_path / "first.txt", + "second": tmp_path / "second.txt" + }, + context="test paths", + ) + + with pytest.raises(ValueError, match="case-insensitively"): + pipeline_module.validate_distinct_file_paths( + { + "first": tmp_path / "Missing.txt", + "second": tmp_path / "missing.txt" + }, + context="test paths", + ) + + +@pytest.mark.asyncio +async def test_backend_neutral_core_rejects_fake_backend_seed_mismatch(tmp_path: Path): + request = _request(tmp_path) + backend = FakeBackend(seed=7) + + with pytest.raises(ValueError, match="backend seed.*effective seed"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_fake_gate_is_derived_from_immutable_optimizer_snapshot(tmp_path: Path): + base_request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + Path(base_request.optimizer_config_path).write_text( + json.dumps({ + "seed": 91, + "gate": { + "max_total_cost": None + } + }), + encoding="utf-8", + ) + request, selected_backend = build_pipeline_request_and_backend( + train_path=base_request.train_path, + val_path=base_request.validation_path, + optimizer_config_path=base_request.optimizer_config_path, + prompt_path=base_request.target_prompt_paths["system_prompt"], + output_dir=base_request.output_dir, + mode="fake", + run_id="gate_snapshot_run", + backend=backend, + ) + Path(base_request.optimizer_config_path).write_text( + json.dumps({ + "seed": 91, + "gate": { + "max_total_cost": 0.0 + } + }), + encoding="utf-8", + ) + + report = await execute_pipeline( + request, + evaluator=selected_backend, + optimizer=selected_backend, + ) + + assert report.selected_candidate is None + assert report.audit["gate_config_snapshot"]["max_total_cost"] == 0.0 + + +@pytest.mark.asyncio +async def test_configured_protected_case_ids_must_exist_in_validation_metadata(tmp_path: Path): + request = _request( + tmp_path, + gate_config={ + "min_val_score_improvement": 0.0, + "protected_case_ids": ["missing"], + "max_score_drop_per_case": 1.0, + "max_total_cost": None, + }, + ) + Path(request.validation_path).write_text( + json.dumps({"eval_cases": [{ + "eval_id": "present", + "session_input": { + "state": {} + } + }]}), + encoding="utf-8", + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="missing validation cases.*missing"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload", + [ + { + "eval_cases": [{ + "eval_id": 7, + "session_input": { + "state": {} + } + }] + }, + { + "eval_cases": [{ + "eval_id": "", + "session_input": { + "state": {} + } + }] + }, + { + "eval_cases": [{ + "eval_id": "case", + "session_input": { + "state": { + "eval_optimize_protected": "yes" + } + } + }] + }, + { + "eval_cases": [{ + "eval_id": "case", + "session_input": { + "state": { + "eval_optimize_protected": None + } + } + }] + }, + { + "cases": [{ + "case_id": 7, + "protected": False + }] + }, + { + "cases": [{ + "case_id": "case", + "protected": "yes" + }] + }, + { + "cases": [{ + "case_id": "case", + "protected": 1 + }] + }, + ], +) +async def test_validation_metadata_rejects_coerced_ids_and_protected_flags( + tmp_path: Path, + payload: dict[str, object], +): + request = _request(tmp_path) + Path(request.validation_path).write_text(json.dumps(payload), encoding="utf-8") + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="case ID|protected.*boolean"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_validation_metadata_rejects_ambiguous_dual_schema(tmp_path: Path): + request = _request(tmp_path) + Path(request.validation_path).write_text( + json.dumps({ + "eval_cases": [], + "cases": [] + }), + encoding="utf-8", + ) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="exactly one.*eval_cases.*cases"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("candidate_ids", [["CON"], ["candidate."], ["Candidate", "candidate"], ["bad/name"]]) +async def test_candidate_ids_are_artifact_safe_and_casefold_unique( + tmp_path: Path, + candidate_ids: list[str], +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=0) + backend.candidates = [ + CandidatePrompt(candidate_id, "safe prompt", "safe", "diff") for candidate_id in candidate_ids + ] + + with pytest.raises(ValueError, match="candidate_id"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_candidate_prompts_must_be_valid_utf8_before_evaluation(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=0) + backend.candidates = [CandidatePrompt("candidate", "bad\ud800prompt", "safe", "diff")] + + with pytest.raises(ValueError, match="valid UTF-8"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert [call[:3] for call in backend.calls] == [ + ("evaluate", "baseline", "train"), + ("evaluate", "baseline", "validation"), + ( + "optimize", + { + "system_prompt": "baseline\n" + }, + Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "optimizer", + ), + ] + + +@pytest.mark.asyncio +async def test_eval_results_reject_non_finite_scores_and_negative_costs(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + baseline = backend.results[("baseline", "train")] + backend.results[("baseline", "train")] = replace( + baseline, + cost=-0.01, + cases=[replace(baseline.cases[0], score=math.nan)], + ) + + with pytest.raises(ValueError, match="finite|non-negative"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["result_score", "result_cost", "case_metric"]) +async def test_eval_results_reject_boolean_numeric_values(tmp_path: Path, field: str): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + baseline = backend.results[("baseline", "train")] + if field == "result_score": + malformed = replace(baseline, score=True) + elif field == "result_cost": + malformed = replace(baseline, cost=True) + else: + malformed = replace( + baseline, + cases=[replace(baseline.cases[0], metrics={"quality": True})], + ) + backend.results[("baseline", "train")] = malformed + + with pytest.raises(ValueError, match="finite number"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_complete_cost_summary_must_equal_components(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend( + candidate_count=1, + optimization_cost=CostSummary(optimizer=0.1, evaluator=0.2, agent=0.3, total=9.0, complete=True), + ) + + with pytest.raises(ValueError, match="total.*components"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_optimizer_rounds_reject_invalid_numeric_values(tmp_path: Path): + request = _request(tmp_path) + invalid_round = OptimizationRound( + round_id=1, + candidate_id="candidate_a", + prompts={"system_prompt": "safe prompt"}, + rationale="bad duration", + metrics={"quality": math.inf}, + cost=CostSummary(), + duration_seconds=-1.0, + ) + backend = _safe_backend(candidate_count=1) + backend.rounds = [invalid_round] + + with pytest.raises(ValueError, match="round.*finite|duration.*non-negative"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bad_value", [Path("private.txt"), {"not-hashable"}, object()]) +async def test_optimizer_raw_summary_must_be_json_compatible( + tmp_path: Path, + bad_value: object, +): + request = _request(tmp_path, update_source=True) + delegate = _safe_backend(candidate_count=1) + backend = RecordingBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + raw_summary={"bad": bad_value}, + ) + + with pytest.raises(ValueError, match="JSON-compatible"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert (tmp_path / "prompt.txt").read_bytes() == b"baseline\n" + + +class SnapshotMutationBackend(RecordingBackend): + + def __init__(self, *, request: PipelineRequest, delegate: RecordingBackend) -> None: + super().__init__( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + self.request = request + self.dataset_paths: list[Path] = [] + self.optimizer_paths: list[tuple[Path, Path, Path]] = [] + self._mutated = False + + async def evaluate(self, **kwargs: Any) -> EvalResult: + dataset_path = Path(kwargs["dataset_path"]) + self.dataset_paths.append(dataset_path) + expected_id = "train_case" if kwargs["split"] == "train" else "validation_case" + payload = json.loads(dataset_path.read_text(encoding="utf-8")) + assert [case["eval_id"] for case in payload["eval_cases"]] == [expected_id] + result = await super().evaluate(**kwargs) + if not self._mutated: + Path(self.request.train_path).write_text('{"eval_cases": [{"eval_id": "changed"}]}', encoding="utf-8") + Path(self.request.optimizer_config_path).write_text('{"seed": 999}', encoding="utf-8") + if self.request.gate_config_path is not None: + Path(self.request.gate_config_path).write_text('{"gate": {"max_total_cost": 0}}', encoding="utf-8") + self._mutated = True + return result + + async def optimize_candidates(self, **kwargs: Any) -> OptimizationResult: + paths = ( + Path(kwargs["train_path"]), + Path(kwargs["validation_path"]), + Path(kwargs["config_path"]), + ) + self.optimizer_paths.append(paths) + assert json.loads(paths[0].read_text(encoding="utf-8"))["eval_cases"][0]["eval_id"] == "train_case" + assert json.loads(paths[1].read_text(encoding="utf-8"))["eval_cases"][0]["eval_id"] == "validation_case" + assert json.loads(paths[2].read_text(encoding="utf-8"))["seed"] == 91 + return await super().optimize_candidates(**kwargs) + + +class SnapshotTamperingBackend(RecordingBackend): + + async def evaluate(self, **kwargs: Any) -> EvalResult: + result = await super().evaluate(**kwargs) + if kwargs["prompt_id"] == "baseline" and kwargs["split"] == "train": + Path(kwargs["dataset_path"]).write_text( + '{"eval_cases": [{"eval_id": "tampered"}]}', + encoding="utf-8", + ) + return result + + +@pytest.mark.asyncio +async def test_backend_cannot_mutate_immutable_input_snapshot(tmp_path: Path): + request = _request(tmp_path, update_source=True) + delegate = _safe_backend(candidate_count=1) + backend = SnapshotTamperingBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + + with pytest.raises(ValueError, match="immutable input snapshot.*train"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert (tmp_path / "prompt.txt").read_bytes() == b"baseline\n" + assert list(tmp_path.glob(".*.snapshot-*")) == [] + + +@pytest.mark.asyncio +async def test_pipeline_uses_immutable_entry_snapshots_and_rejects_original_input_drift(tmp_path: Path): + request = _request(tmp_path, update_source=True) + gate_path = tmp_path / "gate.json" + gate_path.write_text('{"gate": {"max_total_cost": null}}', encoding="utf-8") + request = replace(request, gate_config_path=gate_path) + backend = SnapshotMutationBackend(request=request, delegate=_safe_backend(candidate_count=1)) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + original_paths = { + Path(request.train_path).resolve(), + Path(request.validation_path).resolve(), + Path(request.optimizer_config_path).resolve(), + } + used_paths = {path.resolve() for path in backend.dataset_paths} + used_paths.update(path.resolve() for triple in backend.optimizer_paths for path in triple) + assert used_paths.isdisjoint(original_paths) + assert all(".snapshot-" in path.name for path in used_paths) + assert all( + path.parent in { + Path(request.train_path).resolve().parent, + Path(request.validation_path).resolve().parent, + Path(request.optimizer_config_path).resolve().parent, + } for path in used_paths) + assert report.writeback.status == "rejected" + assert report.audit["writeback_journal"]["state"] == "conflict" + assert "input changed" in (report.writeback.error or "") + assert (tmp_path / "prompt.txt").read_bytes() == b"baseline\n" + assert list(tmp_path.glob(".*.snapshot-*")) == [] + + +@pytest.mark.asyncio +async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_path: Path): + request = _request(tmp_path) + config_payload = { + "seed": 91, + "api_key": "plain-api-key", + "nested": { + "access_token": "plain-token", + "accessToken": "plain-camel-access-token", + "authToken": "plain-camel-auth-token", + "bearerToken": "plain-camel-bearer-token", + "idToken": "plain-camel-id-token", + "githubToken": "plain-camel-github-token", + "passwordValue": "plain-camel-password-value", + "credentials": { + "password": "plain-password" + }, + "github_token": "plain-github-token", + "private_key": "plain-private-key", + "max_tokens": 2048, + "token_budget": 4096, + "cache_path": str(tmp_path / "private-cache"), + }, + } + Path(request.optimizer_config_path).write_text(json.dumps(config_payload), encoding="utf-8") + backend = _safe_backend(candidate_count=1) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + payload = json.dumps(report, default=lambda value: value.__dict__, sort_keys=True) + + assert "plain-api-key" not in payload + assert "plain-token" not in payload + assert "plain-password" not in payload + assert "plain-github-token" not in payload + assert "plain-private-key" not in payload + assert "plain-camel-access-token" not in payload + assert "plain-camel-auth-token" not in payload + assert "plain-camel-bearer-token" not in payload + assert "plain-camel-id-token" not in payload + assert "plain-camel-github-token" not in payload + assert "plain-camel-password-value" not in payload + assert report.audit["config_snapshot"]["nested"]["max_tokens"] == 2048 + assert report.audit["config_snapshot"]["nested"]["token_budget"] == 4096 + assert str(tmp_path.resolve()) not in payload + assert report.run["reproducibility_shell"] == "powershell" + assert "$EXTERNAL" in report.run["reproducibility_command"] + assert (report.audit["config_hash"] == hashlib.sha256( + json.dumps(config_payload, ensure_ascii=False, sort_keys=True, + separators=(",", ":")).encode("utf-8")).hexdigest()) + persisted = (Path(request.output_dir) / "optimization_report.json").read_text(encoding="utf-8") + persisted_markdown = (Path(request.output_dir) / "optimization_report.md").read_text(encoding="utf-8") + config_snapshot = Path(request.output_dir).joinpath( + "runs", + request.run_id, + "config.snapshot.json", + ).read_text(encoding="utf-8") + final_run_dir = Path(request.output_dir) / "runs" / request.run_id + persisted_bytes = persisted.encode("utf-8") + persisted_markdown.encode("utf-8") + persisted_bytes += b"".join(path.read_bytes() for path in final_run_dir.rglob("*") if path.is_file()) + for secret in ( + "plain-api-key", + "plain-token", + "plain-password", + "plain-github-token", + "plain-private-key", + "plain-camel-access-token", + "plain-camel-auth-token", + "plain-camel-bearer-token", + "plain-camel-id-token", + "plain-camel-github-token", + "plain-camel-password-value", + ): + assert secret.encode("utf-8") not in persisted_bytes + redacted_snapshot = json.loads(config_snapshot) + assert redacted_snapshot["nested"]["max_tokens"] == 2048 + assert redacted_snapshot["nested"]["token_budget"] == 4096 + assert "```powershell" in persisted_markdown + assert str(tmp_path.resolve()) not in persisted_markdown + assert hashlib.sha256( + config_snapshot.encode("utf-8")).hexdigest() == report.audit["redacted_config_snapshot_sha256"] + assert report.audit["config_file_sha256"] == hashlib.sha256(Path( + request.optimizer_config_path).read_bytes()).hexdigest() + + +def test_atomic_artifact_write_fsyncs_before_durable_replace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + path = tmp_path / "artifact.json" + events: list[tuple[str, object]] = [] + real_fsync = os.fsync + real_durable_replace = report_module._durable_replace + + def tracked_fsync(fd: int) -> None: + events.append(("fsync", fd)) + real_fsync(fd) + + def tracked_durable_replace(source: Path, target: Path) -> None: + events.append(("durable_replace", Path(target))) + real_durable_replace(source, target) + + monkeypatch.setattr(report_module.os, "fsync", tracked_fsync) + monkeypatch.setattr(report_module, "_durable_replace", tracked_durable_replace) + + report_module._atomic_write_text(path, "durable\n") + + replace_index = next(index for index, event in enumerate(events) if event[0] == "durable_replace") + assert any(event[0] == "fsync" for event in events[:replace_index]) + assert path.read_text(encoding="utf-8") == "durable\n" + assert not list(tmp_path.glob("*.tmp")) + + +def test_powershell_reproducibility_placeholders_remain_expandable(tmp_path: Path): + request = _request(tmp_path) + router_path = tmp_path / "router prompt.txt" + router_path.write_text("router", encoding="utf-8") + request = replace( + request, + target_prompt_paths={ + "system_prompt": request.target_prompt_paths["system_prompt"], + "router_prompt": router_path, + }, + ) + + command = pipeline_module._reproducibility_command(request) + + assert '--output-dir "$OUTPUT_DIR"' in command + assert '"$EXTERNAL/train.evalset.json"' in command + assert '"router_prompt=$EXTERNAL/router prompt.txt"' in command + assert "'$OUTPUT_DIR'" not in command + assert "'$EXTERNAL" not in command + + +@pytest.mark.skipif(os.name != "nt", reason="Windows MoveFileExW contract") +def test_windows_durable_replace_does_not_resolve_target( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + source = tmp_path / "source.tmp" + target = tmp_path / "target.json" + source.write_text("content", encoding="utf-8") + calls: list[tuple[str, str, int]] = [] + + class FakeMoveFileEx: + argtypes: object = None + restype: object = None + + def __call__(self, source_name: str, target_name: str, flags: int) -> int: + calls.append((source_name, target_name, flags)) + return 1 + + fake_move = FakeMoveFileEx() + fake_kernel = type("FakeKernel", (), {"MoveFileExW": fake_move})() + monkeypatch.setattr(report_module.ctypes, "WinDLL", lambda *args, **kwargs: fake_kernel) + monkeypatch.setattr(report_module, "_is_reparse_point", lambda path: False) + real_resolve = Path.resolve + + def reject_target_resolve(path: Path, *args: Any, **kwargs: Any) -> Path: + if path == target: + raise AssertionError("durable replace must not resolve the target leaf") + return real_resolve(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", reject_target_resolve) + + report_module._durable_replace(source, target) + + assert calls + assert calls[0][1] == str(target.absolute()) + + +@pytest.mark.skipif(os.name != "nt", reason="Windows reparse-point contract") +def test_windows_durable_replace_rejects_existing_reparse_target( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + source = tmp_path / "source.tmp" + target = tmp_path / "target.json" + source.write_text("content", encoding="utf-8") + target.write_text("old", encoding="utf-8") + monkeypatch.setattr(report_module, "_is_reparse_point", lambda path: path == target) + monkeypatch.setattr( + report_module.ctypes, + "WinDLL", + lambda *args, **kwargs: pytest.fail("MoveFileExW must not follow a reparse target"), + ) + + with pytest.raises(OSError, match="reparse-point target"): + report_module._durable_replace(source, target) + + +@pytest.mark.asyncio +async def test_prompt_integrity_is_checked_immediately_before_prepare( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=False) + backend = _safe_backend(candidate_count=1) + original_build_report = pipeline_module.build_report + + def mutate_after_report_build(**kwargs: Any): + report = original_build_report(**kwargs) + Path(request.target_prompt_paths["system_prompt"]).write_text( + "late source mutation", + encoding="utf-8", + ) + return report + + monkeypatch.setattr(pipeline_module, "build_report", mutate_after_report_build) + + with pytest.raises(ConcurrentPromptUpdateError, match="source prompt integrity changed"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + run_dir = Path(request.output_dir) / "runs" / request.run_id + assert not (run_dir / "pre_write_report.json").exists() + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +async def test_late_no_write_source_mutation_downgrades_journal_to_unknown( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path, update_source=False) + backend = _safe_backend(candidate_count=1) + original_finalize = pipeline_module.finalize_run_artifacts + + def mutate_after_final_writes(report, paths, *, before_publish): + + def mutate_then_verify(): + Path(request.target_prompt_paths["system_prompt"]).write_text( + "mutation after final artifacts", + encoding="utf-8", + ) + before_publish() + + original_finalize(report, paths, before_publish=mutate_then_verify) + + monkeypatch.setattr(pipeline_module, "finalize_run_artifacts", mutate_after_final_writes) + + with pytest.raises(ConcurrentPromptUpdateError, match="source prompt integrity changed"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + journal_path = (Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" / "writeback_journal.json") + journal = json.loads(journal_path.read_text(encoding="utf-8")) + assert journal["state"] == "unknown" + assert journal["after_hashes"] == {"system_prompt": hashlib.sha256(b"mutation after final artifacts").hexdigest()} + assert not (Path(request.output_dir) / "runs" / request.run_id).exists() + + +@pytest.mark.asyncio +async def test_prepare_writes_completion_journal_after_all_other_artifacts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + writes: list[str] = [] + original = report_module._atomic_write_text + + def tracked(path: Path, content: str) -> None: + writes.append(Path(path).name) + original(Path(path), content) + + monkeypatch.setattr(report_module, "_atomic_write_text", tracked) + + report_module.prepare_run_artifacts(report, tmp_path / "prepare-order") + + assert writes[-1] == "writeback_journal.json" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("identity_error", "message"), + [ + ("prompt_id", "prompt_id"), + ("result_split", "EvalResult split"), + ("case_split", "CaseResult.*split"), + ], +) +async def test_backend_result_identity_must_match_requested_evaluation( + tmp_path: Path, + identity_error: str, + message: str, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + if identity_error == "prompt_id": + baseline_train = backend.results[("baseline", "train")] + backend.results[("baseline", "train")] = replace( + baseline_train, + prompt_id="wrong_prompt", + ) + elif identity_error == "result_split": + candidate_validation = backend.results[("candidate_a", "validation")] + backend.results[("candidate_a", "validation")] = replace( + candidate_validation, + split="train", + ) + else: + candidate_validation = backend.results[("candidate_a", "validation")] + backend.results[("candidate_a", "validation")] = replace( + candidate_validation, + cases=[replace(candidate_validation.cases[0], split="train")], + ) + + with pytest.raises(ValueError, match=message): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +@pytest.mark.asyncio +async def test_async_wrapper_runs_inside_active_event_loop_with_injected_backend(tmp_path: Path): + from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline_async + + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + + report = await run_pipeline_async( + train_path=request.train_path, + val_path=request.validation_path, + optimizer_config_path=request.optimizer_config_path, + prompt_path=request.target_prompt_paths["system_prompt"], + output_dir=request.output_dir, + mode="fake", + trace=True, + run_id="async_wrapper_run", + backend=backend, + ) + + assert report.selected_candidate == "candidate_a" + assert report.run["run_id"] == "async_wrapper_run" + + +@pytest.mark.asyncio +async def test_sync_wrapper_rejects_active_event_loop_with_async_hint(tmp_path: Path): + from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + + with pytest.raises(ValueError, match="await run_pipeline_async"): + run_pipeline( + train_path=request.train_path, + val_path=request.validation_path, + optimizer_config_path=request.optimizer_config_path, + prompt_path=request.target_prompt_paths["system_prompt"], + output_dir=request.output_dir, + mode="fake", + run_id="sync_wrapper_run", + backend=backend, + ) + + +def _request( + tmp_path: Path, + *, + update_source: bool = False, + gate_config: dict[str, object] | None = None, +) -> PipelineRequest: + train_path = tmp_path / "train.evalset.json" + validation_path = tmp_path / "validation.evalset.json" + config_path = tmp_path / "optimizer.json" + prompt_path = tmp_path / "prompt.txt" + train_path.write_text( + '{"eval_cases": [{"eval_id": "train_case", "session_input": {"state": {}}}]}', + encoding="utf-8", + ) + validation_path.write_text( + '{"eval_cases": [{"eval_id": "validation_case", "session_input": {"state": {}}}]}', + encoding="utf-8", + ) + config_path.write_text('{"seed": 91}', encoding="utf-8") + prompt_path.write_bytes(b"baseline\n") + return PipelineRequest( + train_path=train_path, + validation_path=validation_path, + optimizer_config_path=config_path, + output_dir=tmp_path / "out", + target_prompt_paths={"system_prompt": prompt_path}, + gate_config=gate_config or { + "min_val_score_improvement": 0.0, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 1.0, + "max_total_cost": None, + }, + trace=True, + update_source=update_source, + mode="fake", + run_id="ordered_run", + ) + + +def _safe_backend( + *, + candidate_count: int = 2, + candidate_validation_score: float = 1.0, + baseline_train_cost: float = 0.01, + baseline_validation_cost: float = 0.01, + candidate_train_cost: float = 0.01, + candidate_validation_cost: float = 0.01, + optimization_cost: CostSummary | None = None, +) -> RecordingBackend: + candidates = [ + CandidatePrompt("candidate_a", "safe prompt", "safe", "diff a"), + CandidatePrompt("candidate_b", "other prompt", "other", "diff b"), + ][:candidate_count] + results = { + ("baseline", "train"): + _eval( + "baseline", + "train", + ["train_case"], + 0.0, + cost=baseline_train_cost, + failure_category="train_only", + ), + ("baseline", "validation"): + _eval( + "baseline", + "validation", + ["validation_case"], + 0.0, + cost=baseline_validation_cost, + failure_category="validation_only", + ), + } + for candidate in candidates: + results[(candidate.candidate_id, "train")] = _eval( + candidate.candidate_id, + "train", + ["train_case"], + 1.0, + cost=candidate_train_cost, + ) + results[(candidate.candidate_id, "validation")] = _eval( + candidate.candidate_id, + "validation", + ["validation_case"], + candidate_validation_score, + cost=candidate_validation_cost, + ) + return RecordingBackend( + candidates=candidates, + results=results, + optimization_cost=optimization_cost, + ) + + +def _eval( + prompt_id: str, + split: str, + case_ids: list[str], + score: float, + *, + cost: float = 0.01, + failure_category: str | None = None, +) -> EvalResult: + cases = [ + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="output", + hard_failed=score <= 0.0, + failure_category=failure_category if score < 1.0 else None, + ) for case_id in case_ids + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=score, + passed=bool(cases) and all(case.passed for case in cases), + cost=cost, + cases=cases, + ) + + +def _eval_scores( + prompt_id: str, + split: str, + scores: list[tuple[str, float]], +) -> EvalResult: + cases = [ + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="output", + hard_failed=score <= 0.0, + ) for case_id, score in scores + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=round(sum(case.score for case in cases) / len(cases), 6), + passed=all(case.passed for case in cases), + cost=0.01, + cases=cases, + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py b/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py new file mode 100644 index 00000000..b8d45aca --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_report_artifacts.py @@ -0,0 +1,747 @@ +from __future__ import annotations + +import hashlib +import json +import math +import os +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop import pipeline as pipeline_module +from examples.optimization.eval_optimize_loop.eval_loop import report as report_module +from examples.optimization.eval_optimize_loop.eval_loop.pipeline import execute_pipeline +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CostSummary +from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationRound +from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import ( + FailingCandidateBackend, ) +from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import ( + RecordingBackend, ) +from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import _request +from examples.optimization.eval_optimize_loop.tests.test_pipeline_orchestration import _safe_backend + + +class _BaselineCrashBackend(RecordingBackend): + + async def evaluate(self, **kwargs: Any): + artifact_dir = Path(kwargs["artifact_dir"]) + artifact_dir.joinpath("backend-started.txt").write_text("started\n", encoding="utf-8") + if kwargs["prompt_id"] == "baseline" and kwargs["split"] == "train": + raise RuntimeError("simulated backend crash") + return await super().evaluate(**kwargs) + + +def _crashing_backend() -> _BaselineCrashBackend: + delegate = _safe_backend(candidate_count=1) + return _BaselineCrashBackend( + candidates=delegate.candidates, + results=delegate.results, + optimization_cost=delegate.optimization_cost, + ) + + +def _temp_run_dir(request: Any) -> Path: + return Path(request.output_dir) / "runs" / f".{request.run_id}.tmp" + + +def _final_run_dir(request: Any) -> Path: + return Path(request.output_dir) / "runs" / request.run_id + + +def _round(round_id: int, *, rationale: str) -> OptimizationRound: + return OptimizationRound( + round_id=round_id, + candidate_id="candidate_a", + prompts={"system_prompt": "safe prompt"}, + rationale=rationale, + metrics={"train_score": 1.0}, + cost=CostSummary(), + duration_seconds=0.01, + ) + + +def _assert_manifest_records_match_files(run_dir: Path, manifest: dict[str, Any]) -> None: + seen: set[str] = set() + for record in manifest["files"]: + relative_path = record["path"] + assert relative_path not in seen + seen.add(relative_path) + content = (run_dir / relative_path).read_bytes() + assert record["sha256"] == hashlib.sha256(content).hexdigest(), relative_path + assert record["size_bytes"] == len(content), relative_path + + +def _create_directory_link(link: Path, target: Path) -> None: + try: + os.symlink(target, link, target_is_directory=True) + except (NotImplementedError, OSError) as symlink_error: + pytest.skip(f"directory symlinks unavailable: {symlink_error}") + + +def _remove_directory_link(link: Path) -> None: + if link.is_symlink(): + link.unlink() + elif os.path.lexists(link): + os.rmdir(link) + + +async def _source_report(tmp_path: Path, label: str): + source_root = tmp_path / label + source_root.mkdir() + request = _request(source_root) + backend = _safe_backend(candidate_count=1) + return await execute_pipeline(request, evaluator=backend, optimizer=backend) + + +class _FakeRenameAt2: + + def __init__(self, result: int) -> None: + self.result = result + self.calls: list[tuple[Any, ...]] = [] + self.argtypes: list[Any] | None = None + self.restype: Any = None + + def __call__(self, *args: Any) -> int: + self.calls.append(args) + return self.result + + +def _install_fake_renameat2( + monkeypatch: pytest.MonkeyPatch, + *, + result: int, + error_number: int = 0, +) -> _FakeRenameAt2: + real_ctypes = report_module.ctypes + function = _FakeRenameAt2(result) + + class _FakeLibc: + renameat2 = function + + class _CtypesProxy: + + def CDLL(self, name: Any, *, use_errno: bool): + assert name is None + assert use_errno is True + return _FakeLibc() + + def get_errno(self) -> int: + return error_number + + def __getattr__(self, name: str): + return getattr(real_ctypes, name) + + monkeypatch.setattr(report_module, "ctypes", _CtypesProxy()) + return function + + +def test_posix_rename_noreplace_calls_linux_abi_on_success( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + function = _install_fake_renameat2(monkeypatch, result=0) + source = tmp_path / "source" + target = tmp_path / "target" + + report_module._posix_rename_noreplace(source, target) + + assert function.argtypes == [ + report_module.ctypes.c_int, + report_module.ctypes.c_char_p, + report_module.ctypes.c_int, + report_module.ctypes.c_char_p, + report_module.ctypes.c_uint, + ] + assert function.restype is report_module.ctypes.c_int + assert function.calls == [ + (-100, os.fsencode(source), -100, os.fsencode(target), 0x1), + ] + + +def test_posix_rename_noreplace_maps_collision_to_file_exists( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + _install_fake_renameat2( + monkeypatch, + result=-1, + error_number=report_module.errno.EEXIST, + ) + target = tmp_path / "target" + + with pytest.raises(FileExistsError) as caught: + report_module._posix_rename_noreplace(tmp_path / "source", target) + + assert caught.value.errno == report_module.errno.EEXIST + assert caught.value.filename == str(target) + + +@pytest.mark.parametrize("failure", ["missing_symbol", "unsupported_kernel"]) +def test_posix_rename_noreplace_fails_closed_when_unsupported( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: str, +): + real_ctypes = report_module.ctypes + if failure == "missing_symbol": + + class _LibcWithoutRenameAt2: + pass + + class _MissingSymbolProxy: + + def CDLL(self, name: Any, *, use_errno: bool): + return _LibcWithoutRenameAt2() + + def __getattr__(self, name: str): + return getattr(real_ctypes, name) + + monkeypatch.setattr(report_module, "ctypes", _MissingSymbolProxy()) + expected_errno = report_module.errno.ENOTSUP + else: + _install_fake_renameat2( + monkeypatch, + result=-1, + error_number=report_module.errno.ENOSYS, + ) + expected_errno = report_module.errno.ENOSYS + + with pytest.raises(OSError) as caught: + report_module._posix_rename_noreplace(tmp_path / "source", tmp_path / "target") + + assert not isinstance(caught.value, FileExistsError) + assert caught.value.errno == expected_errno + + +def test_posix_publish_race_never_replaces_new_final_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + paths = report_module.reserve_run_artifacts(tmp_path / "out", run_id="racing_run") + evidence = paths.temp_run_dir / "evidence.txt" + evidence.write_bytes(b"reserved temp evidence") + raced_identity: dict[str, int] = {} + + def race_at_atomic_noreplace(source: Path, target: Path) -> None: + assert Path(source) == paths.temp_run_dir + assert Path(target) == paths.final_run_dir + assert not target.exists() + target.mkdir() + raced_identity["inode"] = target.stat().st_ino + raise FileExistsError(17, "final appeared during atomic publish", str(target)) + + real_os = report_module.os + + class _PosixOSProxy: + name = "posix" + + def __getattr__(self, name: str): + return getattr(real_os, name) + + monkeypatch.setattr(report_module, "os", _PosixOSProxy()) + monkeypatch.setattr(report_module, "_fsync_directory", lambda directory: None) + monkeypatch.setattr( + report_module, + "_posix_rename_noreplace", + race_at_atomic_noreplace, + raising=False, + ) + + with pytest.raises(FileExistsError, match="final appeared|already published"): + report_module._durable_publish_directory(paths) + + assert paths.final_run_dir.is_dir() + assert paths.final_run_dir.stat().st_ino == raced_identity["inode"] + assert paths.temp_run_dir.is_dir() + assert evidence.read_bytes() == b"reserved temp evidence" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reserved_name", ["ordered_run", ".ordered_run.tmp"]) +async def test_duplicate_run_id_never_overwrites_final_or_reuses_temp( + tmp_path: Path, + reserved_name: str, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + occupied = Path(request.output_dir) / "runs" / reserved_name + occupied.mkdir(parents=True) + sentinel = occupied / "sentinel.txt" + sentinel.write_bytes(b"keep me exactly") + + with pytest.raises(FileExistsError, match=request.run_id): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert sentinel.read_bytes() == b"keep me exactly" + assert backend.calls == [] + assert Path(request.target_prompt_paths["system_prompt"]).read_bytes() == b"baseline\n" + assert list(tmp_path.glob(".*.snapshot-*")) == [] + + +@pytest.mark.asyncio +async def test_pipeline_rejects_duplicate_round_ids_before_report_artifact_writes(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + backend.rounds = [_round(1, rationale="first"), _round(1, rationale="duplicate")] + + with pytest.raises(ValueError, match=r"duplicate round_id.*1"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + temp_run = _temp_run_dir(request) + assert temp_run.is_dir() + assert not _final_run_dir(request).exists() + for report_artifact in ( + "pre_write_report.json", + "artifact_manifest.json", + "rounds.json", + "rounds", + ): + assert not (temp_run / report_artifact).exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("writer", ["prepare", "write_reports"]) +async def test_direct_report_writers_reject_duplicate_round_ids_before_writing( + tmp_path: Path, + writer: str, +): + source_root = tmp_path / f"source-{writer}" + source_root.mkdir() + source_request = _request(source_root) + source_report = await execute_pipeline( + source_request, + evaluator=_safe_backend(candidate_count=1), + optimizer=_safe_backend(candidate_count=1), + ) + run_id = f"duplicate_rounds_{writer}" + report = replace( + source_report, + run={ + **source_report.run, "run_id": run_id + }, + rounds=[_round(7, rationale="first"), _round(7, rationale="duplicate")], + ) + output_dir = tmp_path / f"direct-{writer}" + + with pytest.raises(ValueError, match=r"duplicate round_id.*7"): + if writer == "prepare": + paths = report_module.reserve_run_artifacts(output_dir, run_id=run_id) + report_module.prepare_run_artifacts(report, paths) + else: + report_module.write_reports(report, output_dir) + + temp_run = output_dir / "runs" / f".{run_id}.tmp" + assert temp_run.is_dir() + assert list(temp_run.iterdir()) == [] + assert not (output_dir / "runs" / run_id).exists() + + +@pytest.mark.asyncio +async def test_all_runtime_backend_artifacts_use_reserved_temp_until_publication(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + temp_run = _temp_run_dir(request) + + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + artifact_dirs = [call[4] for call in backend.calls if call[0] == "evaluate"] + artifact_dirs.extend(call[2] for call in backend.calls if call[0] == "optimize") + assert artifact_dirs + assert all(path.is_relative_to(temp_run) for path in artifact_dirs) + assert not temp_run.exists() + assert _final_run_dir(request).is_dir() + + +@pytest.mark.asyncio +async def test_backend_crash_leaves_temp_evidence_without_final_publication(tmp_path: Path): + request = _request(tmp_path) + backend = _crashing_backend() + + with pytest.raises(RuntimeError, match="simulated backend crash"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + temp_run = _temp_run_dir(request) + assert (temp_run / "evaluations" / "000_baseline" / "train" / "backend-started.txt").is_file() + assert not _final_run_dir(request).exists() + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +async def test_final_artifact_write_failure_cannot_publish_a_partial_run( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + original_write = report_module._atomic_write_text + + def fail_final_markdown(path: Path, content: str) -> None: + path = Path(path) + if path.name == "optimization_report.md" and path.parent == _temp_run_dir(request): + raise OSError("simulated final report write failure") + original_write(path, content) + + monkeypatch.setattr(report_module, "_atomic_write_text", fail_final_markdown) + + with pytest.raises(OSError, match="simulated final report write failure"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert _temp_run_dir(request).is_dir() + assert (_temp_run_dir(request) / "optimization_report.json").is_file() + assert not _final_run_dir(request).exists() + assert not (Path(request.output_dir) / "optimization_report.json").exists() + assert not (Path(request.output_dir) / "optimization_report.md").exists() + + +@pytest.mark.asyncio +async def test_before_publish_failure_refreshes_manifest_without_replacing_original_error(tmp_path: Path, ): + source_root = tmp_path / "callback-source" + source_root.mkdir() + source_request = _request(source_root) + source_report = await execute_pipeline( + source_request, + evaluator=_safe_backend(candidate_count=1), + optimizer=_safe_backend(candidate_count=1), + ) + run_id = "callback_failure" + journal = { + **source_report.audit["writeback_journal"], + "run_id": run_id, + "state": "not_requested", + } + report = replace( + source_report, + run={ + **source_report.run, "run_id": run_id + }, + audit={ + **source_report.audit, "writeback_journal": journal + }, + ) + paths = report_module.reserve_run_artifacts(tmp_path / "callback-output", run_id=run_id) + report_module.prepare_run_artifacts(report, paths) + + class _PromptIntegritySentinel(RuntimeError): + pass + + def mutate_journal_then_fail() -> None: + report_module.persist_writeback_journal( + paths, + { + **journal, + "state": "unknown", + "error": "source prompt integrity changed before publication", + "after_hashes": { + "system_prompt": "changed" + }, + }, + ) + raise _PromptIntegritySentinel("prompt drift sentinel") + + with pytest.raises(_PromptIntegritySentinel, match="prompt drift sentinel") as caught: + report_module.finalize_run_artifacts( + report, + paths, + before_publish=mutate_journal_then_fail, + ) + + assert type(caught.value) is _PromptIntegritySentinel + assert paths.temp_run_dir.is_dir() + assert not paths.final_run_dir.exists() + manifest = json.loads((paths.temp_run_dir / "artifact_manifest.json").read_text(encoding="utf-8")) + _assert_manifest_records_match_files(paths.temp_run_dir, manifest) + journal_record = next(record for record in manifest["files"] if record["path"] == "writeback_journal.json") + journal_bytes = (paths.temp_run_dir / "writeback_journal.json").read_bytes() + assert journal_record["sha256"] == hashlib.sha256(journal_bytes).hexdigest() + assert journal_record["size_bytes"] == len(journal_bytes) + + +@pytest.mark.asyncio +async def test_prepare_rejects_linked_top_level_audit_directory_before_outside_write(tmp_path: Path, ): + source_report = await _source_report(tmp_path, "linked-top-source") + run_id = "linked_top_level" + report = replace(source_report, run={**source_report.run, "run_id": run_id}) + paths = report_module.reserve_run_artifacts(tmp_path / "linked-top-output", run_id=run_id) + outside = tmp_path / "linked-top-outside" + outside.mkdir() + sentinel = outside / "sentinel.txt" + sentinel.write_bytes(b"outside must remain untouched") + linked_component = paths.temp_run_dir / "case_results" + _create_directory_link(linked_component, outside) + + try: + with pytest.raises(OSError, match="unsafe|reparse|directory|artifact"): + report_module.prepare_run_artifacts(report, paths) + finally: + _remove_directory_link(linked_component) + + assert sentinel.read_bytes() == b"outside must remain untouched" + assert {item.name for item in outside.iterdir()} == {"sentinel.txt"} + assert paths.temp_run_dir.is_dir() + assert not paths.final_run_dir.exists() + + +@pytest.mark.asyncio +async def test_prepare_rejects_linked_candidate_directory_before_outside_write(tmp_path: Path): + source_report = await _source_report(tmp_path, "linked-candidate-source") + run_id = "linked_candidate" + report = replace(source_report, run={**source_report.run, "run_id": run_id}) + paths = report_module.reserve_run_artifacts( + tmp_path / "linked-candidate-output", + run_id=run_id, + ) + outside = tmp_path / "linked-candidate-outside" + outside.mkdir() + sentinel = outside / "sentinel.txt" + sentinel.write_bytes(b"candidate outside must remain untouched") + prompt_root = paths.temp_run_dir / "candidate_prompts" + prompt_root.mkdir() + candidate_artifact = report.audit["candidate_artifacts"]["candidate_a"] + linked_candidate = prompt_root / candidate_artifact + _create_directory_link(linked_candidate, outside) + + try: + with pytest.raises(OSError, match="unsafe|reparse|directory|artifact"): + report_module.prepare_run_artifacts(report, paths) + finally: + _remove_directory_link(linked_candidate) + + assert sentinel.read_bytes() == b"candidate outside must remain untouched" + assert {item.name for item in outside.iterdir()} == {"sentinel.txt"} + assert paths.temp_run_dir.is_dir() + assert not paths.final_run_dir.exists() + + +@pytest.mark.asyncio +async def test_prepare_rejects_non_directory_audit_component_without_replacing_it(tmp_path: Path): + source_report = await _source_report(tmp_path, "audit-file-source") + run_id = "audit_component_file" + report = replace(source_report, run={**source_report.run, "run_id": run_id}) + paths = report_module.reserve_run_artifacts(tmp_path / "audit-file-output", run_id=run_id) + component = paths.temp_run_dir / "case_results" + component.write_bytes(b"backend-owned non-directory") + + with pytest.raises(OSError): + report_module.prepare_run_artifacts(report, paths) + + assert component.read_bytes() == b"backend-owned non-directory" + assert not paths.final_run_dir.exists() + + +@pytest.mark.asyncio +async def test_each_nested_audit_write_rechecks_parent_before_next_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + source_report = await _source_report(tmp_path, "audit-parent-swap-source") + run_id = "audit_parent_swap" + report = replace(source_report, run={**source_report.run, "run_id": run_id}) + paths = report_module.reserve_run_artifacts(tmp_path / "audit-parent-swap-output", run_id=run_id) + results_dir = paths.temp_run_dir / "case_results" + parent_became_unsafe = False + original_write = report_module._atomic_write_text + original_is_reparse = report_module._is_reparse_point + + def tracked_write(path: Path, content: str) -> None: + nonlocal parent_became_unsafe + original_write(path, content) + if Path(path) == results_dir / "baseline_train.json": + parent_became_unsafe = True + + def dynamic_reparse(path: Path) -> bool: + if parent_became_unsafe and Path(path) == results_dir: + return True + return original_is_reparse(Path(path)) + + monkeypatch.setattr(report_module, "_atomic_write_text", tracked_write) + monkeypatch.setattr(report_module, "_is_reparse_point", dynamic_reparse) + + with pytest.raises(OSError, match="unsafe|reparse"): + report_module.prepare_run_artifacts(report, paths) + + assert (results_dir / "baseline_train.json").is_file() + assert not (results_dir / "baseline_validation.json").exists() + assert not paths.final_run_dir.exists() + + +@pytest.mark.asyncio +async def test_nan_serialization_failure_stays_in_temp_and_never_publishes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + original_build_report = pipeline_module.build_report + + def report_with_nan(**kwargs: Any): + report = original_build_report(**kwargs) + report.audit["non_finite"] = math.nan + return report + + monkeypatch.setattr(pipeline_module, "build_report", report_with_nan) + + with pytest.raises(ValueError, match="Out of range float values"): + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert _temp_run_dir(request).is_dir() + assert not _final_run_dir(request).exists() + assert not (Path(request.output_dir) / "optimization_report.json").exists() + + +@pytest.mark.asyncio +async def test_fresh_published_run_has_positive_duration_and_only_strict_json(tmp_path: Path): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert report.audit["duration_seconds"] > 0 + for artifact in _final_run_dir(request).rglob("*.json"): + payload = artifact.read_text(encoding="utf-8") + json.loads( + payload, + parse_constant=lambda value: (_ for _ in ()).throw(ValueError(f"non-standard JSON constant: {value}")), + ) + + +@pytest.mark.asyncio +async def test_manifest_covers_complete_audit_set_and_partial_candidate_results(tmp_path: Path): + request = _request(tmp_path) + delegate = _safe_backend(candidate_count=1) + backend = FailingCandidateBackend( + fail_on=("candidate_a", "validation"), + failure=RuntimeError("validation service unavailable"), + delegate=delegate, + ) + backend.rounds = [_round(1, rationale="candidate proposal")] + + report = await execute_pipeline(request, evaluator=backend, optimizer=backend) + + run_dir = _final_run_dir(request) + manifest = json.loads((run_dir / "artifact_manifest.json").read_text(encoding="utf-8")) + _assert_manifest_records_match_files(run_dir, manifest) + artifacts = manifest["artifacts"] + assert manifest["schema_version"] == "eval_optimize_loop.artifacts.v1" + assert manifest["run_id"] == request.run_id + assert set(artifacts["reports"]) == {"prewrite_json", "prewrite_markdown", "final_json", "final_markdown"} + assert set(artifacts["baseline_case_results"]) == {"train", "validation"} + assert set(artifacts["baseline_prompts"]) == {"system_prompt"} + assert len(artifacts["rounds"]) == 1 + assert set(artifacts["audit_records"]) >= { + "audit", + "config_snapshot", + "input_hashes", + "rounds", + "per_case_deltas", + "gate_decisions", + "writeback", + "writeback_journal", + "evaluation_failures", + } + candidate = artifacts["candidates"][0] + assert candidate["artifact_id"] == report.audit["candidate_artifacts"]["candidate_a"] + assert set(candidate["prompt_bundle"]) == {"system_prompt"} + assert set(candidate["case_results"]) == {"train"} + assert candidate["evaluation_failure"] is True + + declared_files = {item["path"] for item in manifest["files"]} + for group in ( + artifacts["reports"], + artifacts["baseline_case_results"], + artifacts["baseline_prompts"], + artifacts["audit_records"], + ): + assert set(group.values()) <= declared_files + assert set(artifacts["rounds"]) <= declared_files + assert json.loads((run_dir / artifacts["rounds"][0]).read_text(encoding="utf-8"))["round_id"] == 1 + assert candidate["diff"] in declared_files + assert set(candidate["prompt_bundle"].values()) <= declared_files + assert set(candidate["case_results"].values()) <= declared_files + assert json.loads((run_dir / artifacts["audit_records"]["evaluation_failures"]).read_text( + encoding="utf-8"))["candidate_a"]["stage"] == "validation" + + +@pytest.mark.asyncio +async def test_convenience_reports_are_exact_copies_created_after_run_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + request = _request(tmp_path) + backend = _safe_backend(candidate_count=1) + copy_events: list[tuple[str, bool, bool]] = [] + original_write_bytes = report_module._atomic_write_bytes + + def observe_copy(path: Path, content: bytes) -> None: + path = Path(path) + if path.parent == Path(request.output_dir) and path.name.startswith("optimization_report"): + copy_events.append(( + path.name, + _final_run_dir(request).is_dir(), + _temp_run_dir(request).exists(), + )) + original_write_bytes(path, content) + + monkeypatch.setattr(report_module, "_atomic_write_bytes", observe_copy) + + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + assert copy_events == [ + ("optimization_report.json", True, False), + ("optimization_report.md", True, False), + ] + for name in ("optimization_report.json", "optimization_report.md"): + assert (Path(request.output_dir) / name).read_bytes() == (_final_run_dir(request) / name).read_bytes() + + +@pytest.mark.asyncio +async def test_runtime_input_hashes_match_exact_entry_bytes(tmp_path: Path): + request = _request(tmp_path) + exact_inputs = { + "train": b'{ "eval_cases" : [{"eval_id":"train_case","session_input":{"state":{}}}]}\r\n', + "validation": b'{"eval_cases":[{"eval_id":"validation_case","session_input":{"state":{}}}]} \n', + "optimizer": b'{\r\n "seed": 91\r\n}\r\n', + } + Path(request.train_path).write_bytes(exact_inputs["train"]) + Path(request.validation_path).write_bytes(exact_inputs["validation"]) + Path(request.optimizer_config_path).write_bytes(exact_inputs["optimizer"]) + prompt_bytes = b"baseline with exact trailing spaces \r\n" + Path(request.target_prompt_paths["system_prompt"]).write_bytes(prompt_bytes) + backend = _safe_backend(candidate_count=1) + + await execute_pipeline(request, evaluator=backend, optimizer=backend) + + hashes = json.loads((_final_run_dir(request) / "input_hashes.json").read_text(encoding="utf-8")) + for role, content in exact_inputs.items(): + assert hashes[role] == hashlib.sha256(content).hexdigest() + assert hashes["target_prompts"]["system_prompt"] == hashlib.sha256(prompt_bytes).hexdigest() + + +def test_committed_example_matches_committed_inputs_without_personal_paths(): + example_root = Path(__file__).resolve().parents[1] + report_path = example_root / "outputs" / "optimization_report.example.json" + payload = json.loads(report_path.read_text(encoding="utf-8")) + input_hashes = payload["audit"]["input_hashes"] + expected_inputs = { + "train": example_root / "data" / "train.evalset.json", + "validation": example_root / "data" / "val.evalset.json", + "optimizer": example_root / "data" / "optimizer.json", + } + + for role, path in expected_inputs.items(): + assert input_hashes[role] == hashlib.sha256(path.read_bytes()).hexdigest() + prompt_path = example_root / "prompts" / "baseline_system_prompt.txt" + assert input_hashes["target_prompts"]["system_prompt"] == hashlib.sha256(prompt_path.read_bytes()).hexdigest() + + assert payload["schema_version"] == "eval_optimize_loop.v2" + assert payload["run"]["run_id"] == "example" + assert payload["audit"]["duration_seconds"] > 0 + assert payload["rounds"] + assert all(round_record["duration_seconds"] > 0 for round_record in payload["rounds"]) + + serialized = json.dumps(payload, ensure_ascii=False) + assert "C:\\Users\\" not in serialized + assert "/Users/" not in serialized + assert "/home/" not in serialized + assert "partial_applied" not in serialized diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py new file mode 100644 index 00000000..c8e7cc83 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -0,0 +1,2324 @@ +from __future__ import annotations + +import builtins +import inspect +import json +import math +import os +import shlex +import sys +import types +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop import backends as backend_module +from examples.optimization.eval_optimize_loop.eval_loop.backends import SDKBackend +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import OptimizationResult +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import _parse_target_prompt_paths +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + + +def test_backend_protocols_expose_unified_async_api(): + assert inspect.iscoroutinefunction(backend_module.EvaluationBackend.evaluate) + assert inspect.iscoroutinefunction(backend_module.OptimizationBackend.optimize_candidates) + assert inspect.iscoroutinefunction(backend_module.FakeBackend.evaluate) + assert inspect.iscoroutinefunction(backend_module.FakeBackend.optimize_candidates) + assert inspect.iscoroutinefunction(backend_module.SDKBackend.evaluate) + assert inspect.iscoroutinefunction(backend_module.SDKBackend.optimize_candidates) + + +@pytest.mark.asyncio +async def test_fake_backend_implements_unified_contract_with_real_trace(tmp_path: Path): + prompt = "baseline prompt" + dataset_path = tmp_path / "fake_train.evalset.json" + dataset_path.write_text( + json.dumps({ + "split": + "train", + "cases": [{ + "case_id": "case_a", + "input": "Mention latency and retries.", + "expectation": { + "type": "rubric", + "must_include": ["latency", "retries"], + }, + }], + }), + encoding="utf-8", + ) + backend = backend_module.FakeBackend(seed=91) + + result = await backend.evaluate( + prompt_id="baseline", + prompts={"system_prompt": prompt}, + dataset_path=dataset_path, + split="train", + trace=True, + artifact_dir=tmp_path / "fake_eval", + ) + + assert result.cases + assert all(case.metrics == {"fake_judge_score": case.score} for case in result.cases) + assert all(case.trace_available is True and case.trace for case in result.cases) + + without_trace = await backend.evaluate( + prompt_id="baseline_without_trace", + prompts={"system_prompt": prompt}, + dataset_path=dataset_path, + split="train", + trace=False, + artifact_dir=tmp_path / "fake_eval_without_trace", + ) + assert all(case.trace_available is False and not case.trace for case in without_trace.cases) + + with pytest.raises(ValueError, match="system_prompt"): + await backend.evaluate( + prompt_id="missing_system_prompt", + prompts={}, + dataset_path=dataset_path, + split="train", + trace=False, + artifact_dir=tmp_path / "missing", + ) + + +@pytest.mark.asyncio +async def test_fake_backend_wraps_candidates_in_complete_optimization_result(tmp_path: Path): + baseline_prompt = DEFAULT_PROMPT.read_text(encoding="utf-8") + baseline_train = EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[ + CaseResult( + case_id="observed_training_failure", + split="train", + score=0.0, + passed=False, + output="not-json", + failure_category="format_violation", + ) + ], + ) + + result = await backend_module.FakeBackend(seed=91).optimize_candidates( + baseline_prompts={"system_prompt": baseline_prompt}, + baseline_train=baseline_train, + failure_summary={"by_category": { + "format_violation": 1 + }}, + train_path=DEFAULT_TRAIN, + validation_path=DEFAULT_VAL, + config_path=DEFAULT_OPTIMIZER_CONFIG, + artifact_dir=tmp_path / "fake_optimize", + ) + + assert isinstance(result, OptimizationResult) + assert result.candidates + assert result.cost.complete is True + assert result.cost.total == 0.0 + assert len(result.rounds) == len(result.candidates) + assert all(candidate.bundle() == {"system_prompt": candidate.prompt} for candidate in result.candidates) + + +@pytest.mark.asyncio +async def test_fake_backend_distributes_measured_batch_proposal_duration( + tmp_path: Path, + monkeypatch, +): + perf_counter_values = iter((10.0, 10.006)) + monkeypatch.setattr( + backend_module, + "time", + types.SimpleNamespace( + perf_counter=lambda: next(perf_counter_values), + get_clock_info=lambda name: types.SimpleNamespace(resolution=1e-9), + ), + raising=False, + ) + + result = await backend_module.FakeBackend(seed=91).optimize_candidates( + baseline_prompts={"system_prompt": "baseline prompt\n"}, + baseline_train=_failed_fake_train_result(), + failure_summary={"by_category": { + "format_violation": 1 + }}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "fake_optimize", + ) + + durations = [round_record.duration_seconds for round_record in result.rounds] + assert durations == pytest.approx([0.003, 0.003]) + assert all(duration > 0 and math.isfinite(duration) for duration in durations) + assert sum(durations) == pytest.approx(0.006) + assert result.raw_summary["proposal_duration_seconds"] == pytest.approx(0.006) + assert result.raw_summary["round_duration_allocation"] == ("equal_share_of_batch_proposal_duration") + + +@pytest.mark.asyncio +async def test_fake_backend_uses_clock_resolution_when_batch_timer_does_not_advance( + tmp_path: Path, + monkeypatch, +): + perf_counter_values = iter((42.0, 42.0)) + monkeypatch.setattr( + backend_module, + "time", + types.SimpleNamespace( + perf_counter=lambda: next(perf_counter_values), + get_clock_info=lambda name: types.SimpleNamespace(resolution=1e-6), + ), + raising=False, + ) + + result = await backend_module.FakeBackend(seed=91).optimize_candidates( + baseline_prompts={"system_prompt": "baseline prompt\n"}, + baseline_train=_failed_fake_train_result(), + failure_summary={"by_category": { + "format_violation": 1 + }}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "fake_optimize", + ) + + durations = [round_record.duration_seconds for round_record in result.rounds] + assert durations == pytest.approx([0.5e-6, 0.5e-6]) + assert all(duration > 0 and math.isfinite(duration) for duration in durations) + assert sum(durations) == pytest.approx(1e-6) + assert result.raw_summary["proposal_duration_seconds"] == pytest.approx(1e-6) + + +@pytest.mark.asyncio +async def test_fake_backend_returns_no_candidates_without_failure_categories(tmp_path: Path): + baseline_prompt = DEFAULT_PROMPT.read_text(encoding="utf-8") + + result = await backend_module.FakeBackend(seed=91).optimize_candidates( + baseline_prompts={"system_prompt": baseline_prompt}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={"failed_case_ids": ["unclassified_training_failure"]}, + train_path=DEFAULT_TRAIN, + validation_path=DEFAULT_VAL, + config_path=DEFAULT_OPTIMIZER_CONFIG, + artifact_dir=tmp_path / "fake_optimize_without_categories", + ) + + assert result.candidates == [] + assert result.rounds == [] + + +@pytest.mark.asyncio +async def test_sdk_backend_maps_rounds_deduplicates_bundles_and_marks_cost_incomplete( + tmp_path: Path, + monkeypatch, +): + round_prompts = { + "system_prompt": "round system", + "router_prompt": "round router", + } + rounds = [ + _sdk_round( + 1, + round_prompts, + acceptance_reason="first proposal", + metric_breakdown={"quality": 0.6}, + round_llm_cost=0.1, + duration_seconds=0.5, + accepted=False, + ), + _sdk_round( + 2, + dict(round_prompts), + acceptance_reason="duplicate proposal", + metric_breakdown={"quality": 0.7}, + round_llm_cost=0.2, + duration_seconds=0.75, + accepted=True, + ), + ] + calls = _install_fake_sdk( + monkeypatch, + best_prompts=round_prompts, + rounds=rounds, + total_llm_cost=0.3, + ) + system_path = tmp_path / "system.txt" + router_path = tmp_path / "router.txt" + system_path.write_text("baseline system", encoding="utf-8") + router_path.write_text("baseline router", encoding="utf-8") + baseline_prompts = { + "system_prompt": "baseline system", + "router_prompt": "baseline router", + } + backend = SDKBackend( + prompt_path=system_path, + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={ + "system_prompt": system_path, + "router_prompt": router_path + }, + ) + + result = await backend.optimize_candidates( + baseline_prompts=baseline_prompts, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={"failed_case_ids": ["case_a"]}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + assert calls["update_source"] is False + assert [candidate.candidate_id for candidate in result.candidates] == ["sdk_round_001"] + assert result.candidates[0].prompt_fields == round_prompts + assert result.candidates[0].bundle() == round_prompts + assert result.candidates[0].rationale == "first proposal" + assert "router_prompt" in result.candidates[0].prompt_diff + assert [round_record.round_id for round_record in result.rounds] == [1, 2] + assert result.rounds[0].metrics == {"quality": 0.6} + assert result.rounds[1].cost.total == 0.2 + assert result.cost.total == 0.3 + assert result.cost.complete is False + assert result.cost.reported_optimizer_cost == 0.3 + assert result.raw_summary["rounds"][1]["acceptance_reason"] == "duplicate proposal" + assert result.raw_summary["rounds"][1]["accepted"] is True + + +@pytest.mark.asyncio +async def test_sdk_backend_appends_best_when_no_round_contains_it(tmp_path: Path, monkeypatch): + round_prompts = {"system_prompt": "round prompt"} + best_prompts = {"system_prompt": "best prompt"} + _install_fake_sdk( + monkeypatch, + best_prompts=best_prompts, + rounds=[_sdk_round(4, round_prompts, acceptance_reason="explored")], + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + result = await SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + assert [candidate.candidate_id for candidate in result.candidates] == ["sdk_round_004", "sdk_best"] + assert result.candidates[1].prompt_fields == best_prompts + + +@pytest.mark.asyncio +async def test_sdk_backend_rejects_duplicate_round_ids(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="best prompt", + rounds=[ + _sdk_round(1, {"system_prompt": "first"}), + _sdk_round(1, {"system_prompt": "second"}), + ], + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="duplicate SDK round id: 1"): + await SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ["FAILED", "CANCELED"]) +async def test_sdk_backend_rejects_unsuccessful_optimize_result( + tmp_path: Path, + monkeypatch, + status: str, +): + from trpc_agent_sdk.evaluation import OptimizeResult + + sdk_result = OptimizeResult( + algorithm="gepa_reflective", + status=status, + finish_reason="error", + stop_reason="user_requested_stop", + error_message="optimizer exploded", + baseline_pass_rate=0.5, + best_pass_rate=0.5, + pass_rate_improvement=0.0, + baseline_prompts={"system_prompt": "baseline"}, + best_prompts={"system_prompt": "baseline"}, + total_rounds=0, + rounds=[], + total_reflection_lm_calls=0, + total_llm_cost=0.0, + duration_seconds=0.1, + started_at="2026-07-04T12:00:00+00:00", + finished_at="2026-07-04T12:00:00.100000+00:00", + ) + _install_fake_sdk(monkeypatch, result_override=sdk_result) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ) + + with pytest.raises(ValueError) as exc_info: + await backend.optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + message = str(exc_info.value) + assert f"status={status}" in message + assert "error_message=optimizer exploded" in message + assert "finish_reason=error" in message + assert "stop_reason=user_requested_stop" in message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field_name", "field_value", "message"), + [ + ("total_llm_cost", -0.01, "total_llm_cost.*non-negative"), + ("duration_seconds", -0.01, "duration_seconds.*non-negative"), + ("baseline_pass_rate", -0.01, "baseline_pass_rate.*between 0 and 1"), + ("best_pass_rate", 1.01, "best_pass_rate.*between 0 and 1"), + ], +) +async def test_sdk_backend_rejects_invalid_top_level_numeric_values( + tmp_path: Path, + monkeypatch, + field_name, + field_value, + message, +): + kwargs = {field_name: field_value, "rounds": []} + _install_fake_sdk(monkeypatch, best_prompt="best", **kwargs) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match=message): + await SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field_name", "field_value", "message"), + [ + ("round_llm_cost", -0.01, "round_llm_cost.*non-negative"), + ("duration_seconds", -0.01, "duration_seconds.*non-negative"), + ("train_pass_rate", -0.01, "train_pass_rate.*between 0 and 1"), + ("validation_pass_rate", 1.01, "validation_pass_rate.*between 0 and 1"), + ], +) +async def test_sdk_backend_rejects_invalid_round_numeric_values( + tmp_path: Path, + monkeypatch, + field_name, + field_value, + message, +): + round_record = _sdk_round(1, {"system_prompt": "candidate"}) + setattr(round_record, field_name, field_value) + _install_fake_sdk( + monkeypatch, + best_prompt="best", + rounds=[round_record], + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match=message): + await SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + +def test_sdk_result_mapping_preserves_metrics_trace_and_expected_label(): + expected_case = EvalCase( + case_id="case_a", + split="validation", + input="question", + expectation={"answer": "expected"}, + expected_failure_category="format_violation", + ) + first_run = _sdk_case_run( + "case_a", + status="FAILED", + run_id=1, + metrics=[ + _sdk_metric("response_match", 0.5, status="FAILED", reason="wrong response"), + _sdk_metric("style", 0.25, status="PASSED"), + ], + output="first output", + user_content="first question", + intermediate_data={"step": 1}, + ) + last_run = _sdk_case_run( + "case_a", + status="FAILED", + run_id=2, + metrics=[ + _sdk_metric("response_match", 1.0, status="PASSED"), + _sdk_metric("style", 0.75, status="PASSED"), + ], + output="last output", + user_content="last question", + intermediate_data={"step": 2}, + ) + + mapped = backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result({"case_a": [first_run, last_run]}), + prompt_id="candidate", + split="validation", + expected_cases=[expected_case], + ) + + case = mapped.cases[0] + assert case.metrics == {"response_match": 0.75, "style": 0.5} + assert case.score == 0.625 + assert case.output == "last output" + assert case.trace_available is True + assert case.trace == { + "user_content": { + "parts": [{ + "text": "last question" + }] + }, + "final_response": { + "parts": [{ + "text": "last output" + }] + }, + "intermediate_data": { + "step": 2 + }, + } + assert case.expected_failure_category == "format_violation" + assert case.failure_category == "final_response_mismatch" + assert case.failure_reason == "wrong response" + assert case.hard_failed is False + + +@pytest.mark.parametrize( + ("sdk_case_ids", "expected_case_ids", "message"), + [ + (["case_a"], ["case_a", "case_b"], "missing.*case_b"), + (["case_a", "case_b"], ["case_a"], "extra.*case_b"), + ], +) +def test_sdk_result_mapping_rejects_case_id_set_mismatch(sdk_case_ids, expected_case_ids, message): + sdk_runs = {case_id: [_sdk_case_run(case_id, status="PASSED", metrics=[])] for case_id in sdk_case_ids} + expected_cases = [ + EvalCase(case_id=case_id, split="validation", input="", expectation={}) for case_id in expected_case_ids + ] + + with pytest.raises(ValueError, match=message): + backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result(sdk_runs), + prompt_id="candidate", + split="validation", + expected_cases=expected_cases, + ) + + +def test_sdk_result_mapping_rejects_duplicate_case_ids_across_eval_sets(): + run = _sdk_case_run("case_a", status="PASSED", metrics=[]) + result = types.SimpleNamespace( + results_by_eval_set_id={ + "set_a": types.SimpleNamespace(eval_results_by_eval_id={"case_a": [run]}), + "set_b": types.SimpleNamespace(eval_results_by_eval_id={"case_a": [run]}), + }) + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + + with pytest.raises(ValueError, match="duplicate.*case_a"): + backend_module._eval_result_from_sdk_result( + result, + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_result_mapping_rejects_non_finite_metric_scores(): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + run = _sdk_case_run( + "case_a", + status="FAILED", + metrics=[_sdk_metric("quality", float("nan"), status="FAILED")], + ) + + with pytest.raises(ValueError, match="finite"): + backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result({"case_a": [run]}), + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ("eval_id", "wrong_case", "internal eval_id.*wrong_case.*case_a"), + ("eval_set_id", "wrong_set", "internal eval_set_id.*wrong_set.*set_a"), + ], +) +def test_sdk_result_mapping_rejects_internal_run_identity_mismatch( + attribute, + value, + message, +): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + run = _sdk_case_run("case_a", status="PASSED", metrics=[]) + setattr(run, attribute, value) + + with pytest.raises(ValueError, match=message): + backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result({"case_a": [run]}), + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_result_mapping_rejects_duplicate_run_ids(): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + runs = [ + _sdk_case_run("case_a", status="PASSED", metrics=[], run_id=1), + _sdk_case_run("case_a", status="PASSED", metrics=[], run_id=1), + ] + + with pytest.raises(ValueError, match="duplicate run_id 1.*case_a"): + backend_module._eval_result_from_sdk_result( + _sdk_evaluate_result({"case_a": runs}), + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_result_mapping_rejects_num_runs_mismatch(): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + run = _sdk_case_run("case_a", status="PASSED", metrics=[]) + result = _sdk_evaluate_result({"case_a": [run]}) + result.results_by_eval_set_id["set_a"].num_runs = 2 + + with pytest.raises(ValueError, match="num_runs=2.*case_a.*1"): + backend_module._eval_result_from_sdk_result( + result, + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_result_mapping_rejects_empty_evaluate_result(): + expected = [EvalCase(case_id="case_a", split="validation", input="", expectation={})] + + with pytest.raises(ValueError, match="EvaluateResult contains no eval set results"): + backend_module._eval_result_from_sdk_result( + types.SimpleNamespace(results_by_eval_set_id={}), + prompt_id="candidate", + split="validation", + expected_cases=expected, + ) + + +def test_sdk_expected_cases_parse_standard_evalset_metadata(tmp_path: Path): + dataset_path = tmp_path / "validation.evalset.json" + case_payload = _sdk_eval_case_payload( + "case-1", + query="query", + expected="expected", + tags=["x"], + protected=True, + ) + case_payload["conversation"].insert( + 0, + { + "invocation_id": "case-1-turn-0", + "user_content": { + "role": "user", + "parts": [{ + "text": "earlier query" + }], + }, + "final_response": { + "role": "model", + "parts": [{ + "text": "earlier expected" + }], + }, + }, + ) + dataset_path.write_text( + json.dumps(_sdk_evalset_payload([case_payload])), + encoding="utf-8", + ) + + expected_cases = backend_module._load_sdk_expected_cases( + dataset_path, + split="validation", + ) + + assert set(expected_cases) == {"case-1"} + expected_case = expected_cases["case-1"] + assert expected_case.input == "query" + assert expected_case.expectation == { + "type": "exact", + "expected": "expected", + "expected_failure_category": "final_response_mismatch", + } + assert expected_case.tags == ["x"] + assert expected_case.protected is True + assert expected_case.expected_failure_category == "final_response_mismatch" + assert expected_case.split == "validation" + + +def test_sdk_expected_cases_reject_duplicate_eval_ids(tmp_path: Path): + dataset_path = tmp_path / "duplicate.evalset.json" + dataset_path.write_text( + json.dumps(_sdk_evalset_payload([ + _sdk_eval_case_payload("case-1"), + _sdk_eval_case_payload("case-1"), + ])), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="duplicate.*case-1"): + backend_module._load_sdk_expected_cases(dataset_path, split="validation") + + +def test_sdk_expected_cases_reject_empty_standard_evalset(tmp_path: Path): + dataset_path = tmp_path / "empty.evalset.json" + dataset_path.write_text( + json.dumps(_sdk_evalset_payload([])), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="eval_cases must not be empty"): + backend_module._load_sdk_expected_cases(dataset_path, split="validation") + + +def test_sdk_expected_cases_require_expectation_metadata(tmp_path: Path): + payload = _sdk_evalset_payload([_sdk_eval_case_payload("case-1")]) + del payload["eval_cases"][0]["session_input"]["state"]["eval_optimize_expectation"] + dataset_path = tmp_path / "missing_expectation.evalset.json" + dataset_path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="case-1.*eval_optimize_expectation"): + backend_module._load_sdk_expected_cases(dataset_path, split="validation") + + +def test_sdk_expected_cases_reject_invalid_eval_cases_shape(tmp_path: Path): + dataset_path = tmp_path / "invalid_shape.evalset.json" + dataset_path.write_text( + json.dumps({ + "eval_set_id": "set", + "eval_cases": {} + }), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="eval_cases.*list"): + backend_module._load_sdk_expected_cases(dataset_path, split="validation") + + +@pytest.mark.asyncio +async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_bytes( + tmp_path: Path, + monkeypatch, +): + from trpc_agent_sdk.evaluation import EvalConfig + from trpc_agent_sdk.evaluation import EvaluationCasesFailed + + dataset_path = tmp_path / "validation.evalset.json" + dataset_path.write_text( + json.dumps( + _sdk_evalset_payload([ + _sdk_eval_case_payload( + "case_a", + query="question", + expected="answer", + expected_failure_category="format_violation", + ) + ])), + encoding="utf-8", + ) + from trpc_agent_sdk.evaluation import EvalSet + + assert EvalSet.model_validate_json(dataset_path.read_text(encoding="utf-8")).eval_cases[0].eval_id == "case_a" + prompt_path = tmp_path / "prompt.txt" + original_bytes = b"original prompt\r\n" + prompt_path.write_bytes(original_bytes) + sdk_result = _sdk_evaluate_result({ + "case_a": [ + _sdk_case_run( + "case_a", + status="PASSED", + metrics=[_sdk_metric("quality", 1.0, status="PASSED")], + output="answer", + user_content="question", + intermediate_data={"tool": "none"}, + ) + ] + }) + calls = _install_fake_agent_evaluator( + monkeypatch, + result=sdk_result, + on_evaluate=lambda: prompt_path.read_text(encoding="utf-8") == "candidate prompt", + evaluation_error=EvaluationCasesFailed("case failed"), + ) + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ) + + mapped = await backend.evaluate( + prompt_id="candidate", + prompts={"system_prompt": "candidate prompt"}, + dataset_path=dataset_path, + split="validation", + trace=False, + artifact_dir=tmp_path / "sdk_eval", + ) + + assert calls["observed_candidate"] is True + assert calls["eval_result_output_dir"] == str(tmp_path / "sdk_eval") + eval_config_path = Path(calls["eval_metrics_file_path_or_dir"]) + eval_config = EvalConfig.model_validate_json(eval_config_path.read_text(encoding="utf-8")) + assert eval_config.criteria == {"final_response_avg_score": 1.0} + assert [metric.metric_name for metric in eval_config.get_eval_metrics()] == ["final_response_avg_score"] + assert prompt_path.read_bytes() == original_bytes + assert mapped.cases[0].trace_available is True + assert mapped.cases[0].expected_failure_category == "format_violation" + + +@pytest.mark.asyncio +async def test_sdk_backend_evaluate_propagates_non_case_failure_even_with_result( + tmp_path: Path, + monkeypatch, +): + dataset_path = tmp_path / "validation.evalset.json" + dataset_path.write_text( + json.dumps(_sdk_evalset_payload([_sdk_eval_case_payload( + "case_a", + query="question", + expected="answer", + )])), + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + original_bytes = b"original prompt\r\n" + prompt_path.write_bytes(original_bytes) + sdk_result = _sdk_evaluate_result({ + "case_a": [ + _sdk_case_run( + "case_a", + status="PASSED", + metrics=[_sdk_metric("quality", 1.0, status="PASSED")], + output="answer", + user_content="question", + ) + ] + }) + _install_fake_agent_evaluator( + monkeypatch, + result=sdk_result, + on_evaluate=lambda: True, + evaluation_error=RuntimeError("network failed"), + ) + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ) + + with pytest.raises(RuntimeError, match="network failed"): + await backend.evaluate( + prompt_id="candidate", + prompts={"system_prompt": "candidate prompt"}, + dataset_path=dataset_path, + split="validation", + trace=False, + artifact_dir=tmp_path / "sdk_eval", + ) + assert prompt_path.read_bytes() == original_bytes + + +@pytest.mark.asyncio +async def test_sdk_backend_evaluate_real_agent_evaluator_smoke( + tmp_path: Path, + monkeypatch, +): + dataset_path = (tmp_path / "validation.evalset.json").resolve() + dataset_path.write_text( + json.dumps(_sdk_evalset_payload([_sdk_eval_case_payload( + "case_a", + query="question", + expected="answer", + )])), + encoding="utf-8", + ) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("original prompt", encoding="utf-8") + calls: list[str] = [] + call_agent_module = types.ModuleType("real_sdk_smoke_call_agent") + + async def call_agent(query: str) -> str: + calls.append(query) + return "answer" + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "real_sdk_smoke_call_agent", call_agent_module) + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="real_sdk_smoke_call_agent:call_agent", + ) + + result = await backend.evaluate( + prompt_id="candidate", + prompts={"system_prompt": "candidate prompt"}, + dataset_path=dataset_path, + split="validation", + trace=False, + artifact_dir=tmp_path / "real_sdk_eval", + ) + + assert calls == ["question"] + assert len(result.cases) == 1 + assert result.cases[0].case_id == "case_a" + assert result.cases[0].passed is True + assert result.cases[0].metrics == {"final_response_avg_score": 1.0} + assert result.cases[0].output == "answer" + assert prompt_path.read_text(encoding="utf-8") == "original prompt" + + +def test_sdk_backend_requires_call_agent_path(tmp_path: Path): + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt") + + with pytest.raises(ValueError, match="--sdk-call-agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_calls_agent_optimizer_and_converts_best_prompt(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + candidates = backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["config_path"].endswith("optimizer.json") + assert calls["update_source"] is False + assert calls["output_dir"].endswith("out") + assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] + assert candidates[0].candidate_id == "sdk_best" + assert candidates[0].prompt == "optimized prompt" + assert candidates[0].prompt_diff.startswith("--- baseline_system_prompt.txt") + assert backend.last_result is not None + assert backend.last_result_summary["baseline_pass_rate"] == 0.5 + + +def test_sdk_backend_default_target_prompt_uses_system_prompt_from_prompt_path(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"system_prompt": "optimized system"}) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline system", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize( + baseline_prompt="baseline system", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] + assert candidates[0].prompt == "optimized system" + + +def test_sdk_backend_router_prompt_only_can_succeed(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"router_prompt": "optimized router"}) + router_path = tmp_path / "router.txt" + router_path.write_text("baseline router", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={ + "router_prompt": router_path + }, + ).optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("router_prompt", str(router_path))] + assert candidates[0].prompt == "## router_prompt\n\noptimized router" + + +def test_sdk_backend_skill_prompt_only_can_succeed(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"skill_prompt": "optimized skill"}) + skill_path = tmp_path / "skill.txt" + skill_path.write_text("baseline skill", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={ + "skill_prompt": skill_path + }, + ).optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("skill_prompt", str(skill_path))] + assert candidates[0].prompt == "## skill_prompt\n\noptimized skill" + + +def test_sdk_backend_missing_registered_best_prompt_field_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompts={"router_prompt": "optimized router"}) + router_path = tmp_path / "router.txt" + skill_path = tmp_path / "skill.txt" + router_path.write_text("baseline router", encoding="utf-8") + skill_path.write_text("baseline skill", encoding="utf-8") + backend = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={ + "router_prompt": router_path, + "skill_prompt": skill_path + }, + ) + + with pytest.raises(ValueError, match="best_prompts.*missing registered target fields.*skill_prompt"): + backend.optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_empty_best_prompts_dict_error_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompts={}) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="best_prompts was empty"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_never_delegates_source_writeback(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + write_source_when_requested=True, + ) + prompt_path = tmp_path / "prompt.txt" + original_bytes = b"baseline\r\n" + prompt_path.write_bytes(original_bytes) + + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + update_source=True, + ) + backend.optimize( + baseline_prompt="baseline\n", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["update_source"] is False + assert prompt_path.read_bytes() == original_bytes + assert "update_source" not in vars(backend) + + +@pytest.mark.asyncio +async def test_sdk_backend_detects_source_change_at_snapshot_boundary( + tmp_path: Path, + monkeypatch, +): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + real_snapshot = backend_module.snapshot_prompt_files + + def snapshot_after_external_change(paths): + prompt_path.write_text("external change", encoding="utf-8") + return real_snapshot(paths) + + monkeypatch.setattr( + backend_module, + "snapshot_prompt_files", + snapshot_after_external_change, + ) + backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ) + + with pytest.raises(ValueError, match="baseline prompt bundle.*source"): + await backend.optimize_candidates( + baseline_prompts={"system_prompt": "baseline"}, + baseline_train=_empty_eval_result("baseline", "train"), + failure_summary={}, + train_path=tmp_path / "train.evalset.json", + validation_path=tmp_path / "validation.evalset.json", + config_path=tmp_path / "optimizer.json", + artifact_dir=tmp_path / "sdk_optimize", + ) + + assert "update_source" not in calls + assert prompt_path.read_text(encoding="utf-8") == "external change" + + +def test_sdk_backend_call_agent_import_failure_names_target(tmp_path: Path): + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="missing.module:call_agent") + + with pytest.raises(ValueError, match="missing.module:call_agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_call_agent_must_be_callable(tmp_path: Path, monkeypatch): + call_agent_module = types.ModuleType("fake_call_agent_module") + call_agent_module.call_agent = "not callable" + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="--sdk-call-agent.*fake_call_agent_module:call_agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_sdk_import_failure_is_clear(tmp_path: Path, monkeypatch): + call_agent_module = types.ModuleType("fake_call_agent_module") + + async def call_agent(query: str) -> str: + return query + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + + real_import = builtins.__import__ + + def fail_sdk_import(name, globals=None, locals=None, fromlist=(), level=0): + if name == "trpc_agent_sdk.evaluation": + raise ImportError("forced sdk import failure") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", fail_sdk_import) + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="AgentOptimizer/TargetPrompt"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_empty_best_prompt_error_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="contained empty registered target fields.*system_prompt"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +@pytest.mark.asyncio +async def test_sdk_backend_sync_optimize_rejects_active_event_loop(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="optimize_async"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +@pytest.mark.asyncio +async def test_sdk_backend_async_api_works_inside_active_event_loop(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + candidates = await backend.optimize_async( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert candidates[0].candidate_id == "sdk_best" + + +def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + gate_path = _write_gate_config( + tmp_path, + min_val_score_improvement=0.01, + max_total_cost=None, + ) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + run_id="sdk_test_run", + ) + + output_dir = tmp_path / "sdk_run" + payload = json.loads((output_dir / "optimization_report.json").read_text(encoding="utf-8")) + markdown = (output_dir / "optimization_report.md").read_text(encoding="utf-8") + assert report.run["mode"] == "sdk" + assert report.run["update_source"] is False + assert report.selected_candidate == "sdk_best" + assert report.baseline_validation.score == 0.5 + assert report.candidates[0]["validation_result"].score == 0.75 + assert report.gate_decisions[0].validation_score_delta == 0.25 + assert report.gate_decisions[0].candidate_cost == 0.0 + assert report.gate_decisions[0].gate_status == "applied" + assert report.gate_decisions[0].not_applied_checks == [] + assert report.audit["duration_seconds"] > 0 + assert report.audit["total_run_cost"] is None + assert report.audit["known_run_cost"] == 0.123 + assert report.audit["total_run_cost_complete"] is False + assert report.audit["cost"]["total"] is None + assert report.audit["cost"]["complete"] is False + assert report.audit["cost"]["reported_optimizer_cost"] == 0.123 + assert payload["cost_summary"]["reported_optimizer_cost"] == 0.123 + assert payload["cost_summary"]["complete"] is False + assert "Reported optimizer cost (incomplete; not total run cost): 0.123" in markdown + assert "Total cost:" not in markdown + assert report.audit["sdk_result_summary"]["status"] == "SUCCEEDED" + assert report.audit["sdk_result_summary"]["baseline_metric_breakdown"] == {"exact_match": 0.5} + assert report.audit["sdk_result_summary"]["best_metric_breakdown"] == {"exact_match": 0.75} + assert report.audit["sdk_result_summary"]["metric_thresholds"] == {"exact_match": 0.7} + assert report.audit["sdk_result_summary"]["total_token_usage"] == { + "prompt": 100, + "completion": 25, + "total": 125, + } + assert report.audit["sdk_result_summary"]["rounds"][0]["validation_pass_rate"] == 0.75 + assert report.audit["sdk_result_availability"] == { + "aggregate_validation_result": True, + "full_train_eval_result": True, + "full_per_case_validation_delta": True, + } + assert "partial_applied" not in payload + assert "sdk_best (accepted)" in markdown + assert "complete AgentEvaluator-compatible reevaluation" in markdown + assert (output_dir / "runs" / "sdk_test_run" / "input_hashes.json").is_file() + candidate_artifact = report.audit["candidate_artifacts"]["sdk_best"] + assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / f"{candidate_artifact}.diff").is_file() + assert calls["update_source"] is False + assert calls["output_dir"].endswith("runs\\.sdk_test_run.tmp\\optimizer") or calls["output_dir"].endswith( + "runs/.sdk_test_run.tmp/optimizer") + assert calls["evaluation_count"] == 4 + command = report.run["reproducibility_command"] + assert "--sdk-call-agent fake_call_agent_module:call_agent" in command + command_args = shlex.split(command) + gate_index = command_args.index("--gate-config") + assert command_args[gate_index + 1] == "$EXTERNAL/wrapper_gate.json" + assert str(gate_path.parent.resolve()) not in command + + +def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + train_path = tmp_path / "sdk_train.evalset.json" + val_path = tmp_path / "sdk_val.evalset.json" + optimizer_path = tmp_path / "sdk_optimizer.json" + prompt_path = tmp_path / "system_prompt.txt" + train_path.write_text( + json.dumps(_sdk_evalset_payload([_sdk_eval_case_payload("sdk_train_case")])), + encoding="utf-8", + ) + val_path.write_text( + json.dumps(_sdk_evalset_payload([_sdk_eval_case_payload("sdk_val_case")])), + encoding="utf-8", + ) + optimizer_path.write_text( + json.dumps({ + "seed": "sdk-owned-seed", + "optimize": { + "algorithm": { + "name": "gepa_reflective" + } + } + }), + encoding="utf-8", + ) + prompt_path.write_text("baseline", encoding="utf-8") + + report = run_pipeline( + mode="sdk", + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=_write_gate_config( + tmp_path, + min_val_score_improvement=0.01, + max_total_cost=None, + ), + ) + + assert report.run["mode"] == "sdk" + assert report.run["train_cases"] == 1 + assert report.selected_candidate == "sdk_best" + + +def test_run_pipeline_mode_sdk_default_run_id_uses_utc_timestamp_and_random_suffix( + tmp_path: Path, + monkeypatch, +): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + started_at="2026-07-04T12:34:56+00:00", + ) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + assert report.run["run_id"].startswith("eval_optimize_loop_sdk_") + assert report.run["run_id"] != "eval_optimize_loop_sdk_20260704T123456Z" + assert (tmp_path / "sdk_run" / "runs" / report.run["run_id"]).is_dir() + assert f"--run-id {report.run['run_id']}" in report.run["reproducibility_command"] + + +def test_run_pipeline_mode_sdk_default_run_ids_are_unique(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + started_at="2026-07-04T12:34:56+00:00", + ) + + first = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + second = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + assert first.run["run_id"].startswith("eval_optimize_loop_sdk_") + assert second.run["run_id"].startswith("eval_optimize_loop_sdk_") + assert first.run["run_id"] != second.run["run_id"] + assert (tmp_path / "sdk_run" / "runs" / first.run["run_id"]).is_dir() + assert (tmp_path / "sdk_run" / "runs" / second.run["run_id"]).is_dir() + + +def test_run_pipeline_mode_sdk_explicit_run_id_is_stable_and_exclusive(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + first = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id="valid_20260704-1.ok", + ) + with pytest.raises(FileExistsError, match="run ID.*already reserved or published"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id="valid_20260704-1.ok", + ) + + assert first.run["run_id"] == "valid_20260704-1.ok" + + +def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_gate( + tmp_path: Path, + monkeypatch, +): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.505, + pass_rate_improvement=0.005, + total_llm_cost=0.123, + ) + optimizer_path = _write_sdk_optimizer_config(tmp_path) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=optimizer_path, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.gate_status == "applied" + assert decision.not_applied_checks == [] + assert decision.validation_score_delta == 0.005 + assert any("validation improvement" in reason for reason in decision.reasons) + assert any("cost_unavailable" in reason for reason in decision.reasons) + + +def test_run_pipeline_mode_sdk_custom_gate_uses_full_validation_evaluation( + tmp_path: Path, + monkeypatch, +): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.75, + pass_rate_improvement=0.9, + total_llm_cost=0.123, + ) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.3, max_total_cost=None) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.validation_score_delta == 0.25 + assert report.audit["sdk_result_summary"]["pass_rate_improvement"] == 0.9 + assert any("validation improvement" in reason for reason in decision.reasons) + + +def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.75, + pass_rate_improvement=0.25, + total_llm_cost=2.0, + ) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.01, max_total_cost=0.05) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.gate_status == "applied" + assert decision.total_run_cost == 2.0 + assert any("cost_unavailable" in reason for reason in decision.reasons) + + +def test_sdk_pipeline_skips_disabled_cost_gate_but_keeps_quality_threshold( + tmp_path: Path, + monkeypatch, +): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.505, + pass_rate_improvement=0.9, + total_llm_cost=1000.0, + ) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=_write_gate_config( + tmp_path, + min_val_score_improvement=0.01, + max_total_cost=None, + ), + ) + decision = report.gate_decisions[0] + + assert decision.accepted is False + assert decision.validation_score_delta == 0.005 + assert any("validation improvement" in reason for reason in decision.reasons) + assert not any("cost" in reason for reason in decision.reasons) + assert decision.total_run_cost == 1000.0 + + +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + ("min_val_score_improvement", True), + ("max_total_cost", float("nan")), + ("max_total_cost", float("inf")), + ], +) +def test_run_pipeline_mode_sdk_rejects_invalid_gate_numbers( + tmp_path: Path, + monkeypatch, + field_name, + field_value, +): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + gate = {"min_val_score_improvement": 0.01, "max_total_cost": 1.0} + gate[field_name] = field_value + gate_path = tmp_path / "bad_gate.json" + gate_path.write_text(json.dumps({"gate": gate}), encoding="utf-8") + + with pytest.raises(ValueError, match=f"--gate-config.*{field_name}"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + +@pytest.mark.parametrize("run_id", ["../../escape", "a/b", "", ".", "..", "has space", "a\\b"]) +def test_run_pipeline_rejects_invalid_run_id(tmp_path: Path, monkeypatch, run_id): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + with pytest.raises(ValueError, match="--run-id") as exc_info: + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id=run_id, + ) + assert repr(run_id) in str(exc_info.value) + + +@pytest.mark.parametrize( + "field_name", + ["../router", "router/prompt", "router prompt", "router.prompt", "router-prompt", "", " router_prompt"], +) +def test_run_pipeline_mode_sdk_rejects_invalid_target_prompt_field_names( + tmp_path: Path, + monkeypatch, + field_name, +): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="--target-prompt") as exc_info: + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + target_prompts=[f"{field_name}={prompt_path}"], + ) + assert repr(field_name) in str(exc_info.value) + + +def test_target_prompt_paths_reject_same_resolved_file(tmp_path: Path): + prompt_path = tmp_path / "prompt.txt" + equivalent_path = tmp_path / "nested" / ".." / prompt_path.name + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError) as exc_info: + _parse_target_prompt_paths( + [ + f"system_prompt={prompt_path}", + f"router_prompt={equivalent_path}", + ], + default_prompt_path=prompt_path, + ) + + assert "--target-prompt fields must be different physical files" in str(exc_info.value) + + +def test_target_prompt_paths_reject_hardlink_aliases(tmp_path: Path): + prompt_path = tmp_path / "prompt.txt" + hardlink_path = tmp_path / "prompt-hardlink.txt" + prompt_path.write_text("baseline", encoding="utf-8") + os.link(prompt_path, hardlink_path) + assert prompt_path.samefile(hardlink_path) + + with pytest.raises(ValueError, match="different physical files"): + _parse_target_prompt_paths( + [ + f"system_prompt={prompt_path}", + f"router_prompt={hardlink_path}", + ], + default_prompt_path=prompt_path, + ) + + +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + ("pass_rate_improvement", float("nan")), + ("total_llm_cost", float("inf")), + ("best_pass_rate", "bad"), + ], +) +def test_run_pipeline_mode_sdk_rejects_non_finite_or_bad_numeric_summary( + tmp_path: Path, + monkeypatch, + field_name, + field_value, +): + kwargs = { + "baseline_pass_rate": 0.5, + "best_pass_rate": 0.75, + "pass_rate_improvement": 0.25, + "total_llm_cost": 0.123, + } + kwargs[field_name] = field_value + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt", **kwargs) + + with pytest.raises(ValueError, match=f"SDK OptimizeResult field {field_name} must be a finite number"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + +def test_run_pipeline_mode_sdk_does_not_pass_wrapper_gate_config_to_agent_optimizer( + tmp_path: Path, + monkeypatch, +): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + optimizer_path = _write_sdk_optimizer_config(tmp_path) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.5, max_total_cost=0.05) + + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=optimizer_path, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + optimizer_snapshot = Path(calls["config_path"]) + assert optimizer_snapshot.resolve() != optimizer_path.resolve() + assert optimizer_snapshot.parent.resolve() == optimizer_path.parent.resolve() + assert optimizer_snapshot.name.endswith(f".snapshot-{optimizer_path.name}") + assert "gate" not in calls["config_payload"] + assert json.loads(gate_path.read_text(encoding="utf-8"))["gate"]["max_total_cost"] == 0.05 + + +def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk( + monkeypatch, + best_prompts={ + "system_prompt": "optimized system", + "router_prompt": "optimized router", + "skill_prompt": "optimized skill", + }, + ) + system_path = tmp_path / "system.txt" + router_path = tmp_path / "router.txt" + skill_path = tmp_path / "skill.txt" + system_path.write_text("baseline system", encoding="utf-8") + router_path.write_text("baseline router", encoding="utf-8") + skill_path.write_text("baseline skill", encoding="utf-8") + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=system_path, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + target_prompts=[ + f"system_prompt={system_path}", + f"router_prompt={router_path}", + f"skill_prompt={skill_path}", + ], + gate_config_path=_write_gate_config(tmp_path, min_val_score_improvement=0.01, max_total_cost=1.0), + run_id="sdk_multi_target", + ) + + assert calls["target_prompt"].paths == [ + ("system_prompt", str(system_path)), + ("router_prompt", str(router_path)), + ("skill_prompt", str(skill_path)), + ] + assert report.audit["sdk_result_summary"]["best_prompts"] == { + "system_prompt": "optimized system", + "router_prompt": "optimized router", + "skill_prompt": "optimized skill", + } + assert "router_prompt" in report.candidates[0]["candidate"].prompt_diff + assert set(report.audit["candidate_prompt_hashes"]["sdk_best"]) == { + "system_prompt", + "router_prompt", + "skill_prompt", + } + run_dir = tmp_path / "sdk_run" / "runs" / "sdk_multi_target" + candidate_artifact = report.audit["candidate_artifacts"]["sdk_best"] + assert run_dir.joinpath( + "candidate_prompts", + candidate_artifact, + "system_prompt.txt", + ).read_text(encoding="utf-8") == "optimized system" + assert run_dir.joinpath( + "candidate_prompts", + candidate_artifact, + "router_prompt.txt", + ).read_text(encoding="utf-8") == "optimized router" + assert run_dir.joinpath( + "candidate_prompts", + candidate_artifact, + "skill_prompt.txt", + ).read_text(encoding="utf-8") == "optimized skill" + input_hashes = json.loads((run_dir / "input_hashes.json").read_text(encoding="utf-8")) + assert set(input_hashes["target_prompts"]) == { + "system_prompt", + "router_prompt", + "skill_prompt", + } + assert report.audit["gate_config_hash"] + assert report.audit["sdk_result_availability"]["full_train_eval_result"] is True + assert calls["evaluation_count"] == 4 + command = report.run["reproducibility_command"] + assert "--sdk-call-agent fake_call_agent_module:call_agent" in command + assert "--target-prompt" in command + assert "router_prompt=$EXTERNAL/router.txt" in command + assert str(tmp_path.resolve()) not in command + assert "--gate-config" in command + + +def test_run_pipeline_mode_sdk_keeps_source_writeback_outside_backend(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "system_prompt.txt" + prompt_path.write_bytes(b"baseline prompt\r\n") + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=prompt_path, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + update_source=True, + gate_config_path=_write_gate_config( + tmp_path, + min_val_score_improvement=0.01, + max_total_cost=None, + ), + ) + + assert report.run["update_source"] is True + assert calls["update_source"] is False + assert report.writeback.status == "applied" + assert prompt_path.read_text(encoding="utf-8") == "optimized prompt" + assert "--update-source" in report.run["reproducibility_command"] + + +def test_run_pipeline_mode_sdk_missing_call_agent_is_not_fake_fallback(tmp_path: Path): + with pytest.raises(ValueError, match="--sdk-call-agent"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + ) + + +def _install_fake_sdk( + monkeypatch, + *, + best_prompt: str | None = None, + best_prompts: dict[str, str] | None = None, + status: str = "SUCCEEDED", + baseline_pass_rate: float = 0.5, + best_pass_rate: float = 0.75, + pass_rate_improvement: float = 0.25, + total_llm_cost: float = 0.123, + duration_seconds: float = 12.3, + started_at: str | None = None, + rounds: list[object] | None = None, + write_source_when_requested: bool = False, + result_override: object | None = None, +): + calls = {} + + class FakeTargetPrompt: + + def __init__(self): + self.paths = [] + + def add_path(self, name, path): + self.paths.append((name, path)) + return self + + class FakeAgentOptimizer: + + @staticmethod + async def optimize(**kwargs): + calls.update(kwargs) + config_path = Path(kwargs["config_path"]) + if config_path.is_file(): + calls["config_payload"] = json.loads(config_path.read_text(encoding="utf-8")) + if result_override is not None: + return result_override + if write_source_when_requested and kwargs.get("update_source"): + for _, path in kwargs["target_prompt"].paths: + Path(path).write_bytes(b"optimizer mutated source") + result_prompts = best_prompts if best_prompts is not None else { + "system_prompt": "optimized prompt" if best_prompt is None else best_prompt + } + effective_rounds = rounds if rounds is not None else [ + _sdk_round( + 1, + {}, + acceptance_reason="accepted", + validation_pass_rate=best_pass_rate, + accepted=True, + failed_case_ids=["case_a"], + round_llm_cost=total_llm_cost, + duration_seconds=duration_seconds, + ) + ] + return types.SimpleNamespace( + best_prompts=result_prompts, + status=status, + baseline_pass_rate=baseline_pass_rate, + best_pass_rate=best_pass_rate, + pass_rate_improvement=pass_rate_improvement, + baseline_metric_breakdown={"exact_match": baseline_pass_rate}, + best_metric_breakdown={"exact_match": best_pass_rate}, + metric_thresholds={"exact_match": 0.7}, + total_llm_cost=total_llm_cost, + total_token_usage={ + "prompt": 100, + "completion": 25, + "total": 125 + }, + duration_seconds=duration_seconds, + started_at=started_at, + total_rounds=len(effective_rounds), + rounds=effective_rounds, + ) + + class FakeEvalConfig: + + def __init__(self, **kwargs): + self.payload = kwargs + + def model_dump_json(self, indent=2): + return json.dumps(self.payload, indent=indent) + + class FakeEvaluationCasesFailed(Exception): + pass + + class FakeAgentEvaluator: + + @staticmethod + def get_executer(dataset_path, **kwargs): + evaluation_index = int(calls.get("evaluation_count", 0)) + calls["evaluation_count"] = evaluation_index + 1 + calls.setdefault("evaluation_calls", []).append({ + "dataset_path": dataset_path, + **kwargs, + }) + payload = json.loads(Path(dataset_path).read_text(encoding="utf-8")) + if "eval_cases" in payload: + case_ids = [str(case["eval_id"]) for case in payload["eval_cases"]] + else: + case_ids = [str(case.get("case_id") or case.get("id")) for case in payload["cases"]] + is_baseline = evaluation_index % 4 < 2 + score = baseline_pass_rate if is_baseline else best_pass_rate + result = _sdk_evaluate_result({ + case_id: [ + _sdk_case_run( + case_id, + status="PASSED", + metrics=[_sdk_metric("final_response_avg_score", score, status="PASSED")], + output="evaluated output", + ) + ] + for case_id in case_ids + }) + + class FakeExecuter: + + async def evaluate(self): + return None + + def get_result(self): + return result + + return FakeExecuter() + + fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") + fake_eval_module.AgentOptimizer = FakeAgentOptimizer + fake_eval_module.TargetPrompt = FakeTargetPrompt + fake_eval_module.AgentEvaluator = FakeAgentEvaluator + fake_eval_module.EvalConfig = FakeEvalConfig + fake_eval_module.EvaluationCasesFailed = FakeEvaluationCasesFailed + monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) + + call_agent_module = types.ModuleType("fake_call_agent_module") + + async def call_agent(query: str) -> str: + return query + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + return calls + + +def _empty_eval_result(prompt_id: str, split: str) -> EvalResult: + return EvalResult( + prompt_id=prompt_id, + split=split, + score=0.0, + passed=False, + cost=0.0, + cases=[], + ) + + +def _failed_fake_train_result() -> EvalResult: + return EvalResult( + prompt_id="baseline", + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[ + CaseResult( + case_id="observed_training_failure", + split="train", + score=0.0, + passed=False, + output="not-json", + failure_category="format_violation", + ) + ], + ) + + +def _sdk_round( + round_id: int, + candidate_prompts: dict[str, str], + *, + acceptance_reason: str = "", + metric_breakdown: dict[str, float] | None = None, + validation_pass_rate: float = 0.0, + round_llm_cost: float = 0.0, + duration_seconds: float = 0.0, + accepted: bool = False, + failed_case_ids: list[str] | None = None, +): + return types.SimpleNamespace( + round=round_id, + optimized_field_names=list(candidate_prompts), + candidate_prompts=dict(candidate_prompts), + train_pass_rate=0.0, + validation_pass_rate=validation_pass_rate, + metric_breakdown=dict(metric_breakdown or {}), + accepted=accepted, + acceptance_reason=acceptance_reason, + failed_case_ids=list(failed_case_ids or []), + failed_cases_truncated=0, + per_field_diagnosis={}, + reflection_lm_calls=1, + round_llm_cost=round_llm_cost, + round_token_usage={ + "prompt": 0, + "completion": 0, + "total": 0 + }, + started_at="2026-07-04T12:00:00+00:00", + duration_seconds=duration_seconds, + kind="reflective", + train_minibatch_size=0, + train_subsample_parent_score=None, + train_subsample_candidate_score=None, + skip_reason=None, + error_message=None, + budget_used=3, + budget_total=10, + extras={}, + ) + + +def _sdk_metric(metric_name: str, score: float, *, status: str, reason: str | None = None): + return types.SimpleNamespace( + metric_name=metric_name, + threshold=0.5, + score=score, + eval_status=status, + details=types.SimpleNamespace(reason=reason, score=score, rubric_scores=None), + ) + + +def _sdk_case_run( + case_id: str, + *, + status: str, + metrics: list[object], + run_id: int | None = 1, + output: str | None = None, + user_content: str | None = None, + intermediate_data: object | None = None, +): + per_invocation = [] + if output is not None or user_content is not None or intermediate_data is not None: + actual_invocation = types.SimpleNamespace( + invocation_id=f"{case_id}_invocation", + user_content={"parts": [{ + "text": user_content or "" + }]}, + final_response={"parts": [{ + "text": output or "" + }]} if output is not None else None, + intermediate_data=intermediate_data, + creation_timestamp=0.0, + ) + per_invocation.append( + types.SimpleNamespace( + actual_invocation=actual_invocation, + expected_invocation=None, + eval_metric_results=list(metrics), + )) + return types.SimpleNamespace( + eval_set_id="set_a", + eval_id=case_id, + run_id=run_id, + final_eval_status=status, + error_message=None if status == "PASSED" else "case failed", + overall_eval_metric_results=list(metrics), + eval_metric_result_per_invocation=per_invocation, + session_id=f"session_{case_id}", + user_id="test_user", + session_details=None, + ) + + +def _sdk_evaluate_result(runs_by_case_id: dict[str, list[object]]): + return types.SimpleNamespace( + results_by_eval_set_id={ + "set_a": + types.SimpleNamespace( + eval_results_by_eval_id=dict(runs_by_case_id), + num_runs=max((len(runs) for runs in runs_by_case_id.values()), default=1), + ) + }) + + +def _sdk_evalset_payload(eval_cases: list[dict[str, object]]) -> dict[str, object]: + return { + "eval_set_id": "set", + "eval_cases": eval_cases, + } + + +def _sdk_eval_case_payload( + eval_id: str, + *, + query: str = "query", + expected: str = "expected", + expected_failure_category: str = "final_response_mismatch", + tags: list[str] | None = None, + protected: bool = False, +) -> dict[str, object]: + return { + "eval_id": + eval_id, + "conversation": [{ + "invocation_id": f"{eval_id}-turn-1", + "user_content": { + "role": "user", + "parts": [{ + "text": query + }], + }, + "final_response": { + "role": "model", + "parts": [{ + "text": expected + }], + }, + }], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "test-user", + "state": { + "eval_optimize_expectation": { + "type": "exact", + "expected": expected, + "expected_failure_category": expected_failure_category, + }, + "eval_optimize_tags": list(tags or []), + "eval_optimize_protected": protected, + }, + }, + } + + +def _install_fake_agent_evaluator( + monkeypatch, + *, + result: object, + on_evaluate, + evaluation_error: BaseException | None, +): + calls: dict[str, object] = {} + + class FakeExecuter: + + async def evaluate(self): + calls["observed_candidate"] = bool(on_evaluate()) + if evaluation_error is not None: + raise evaluation_error + + def get_result(self): + return result + + class FakeAgentEvaluator: + + @staticmethod + def get_executer(dataset_path, **kwargs): + calls["dataset_path"] = dataset_path + calls.update(kwargs) + return FakeExecuter() + + from trpc_agent_sdk.evaluation._agent_evaluator import _EvaluationCasesFailed + from trpc_agent_sdk.evaluation._eval_config import EvalConfig + + fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") + fake_eval_module.AgentEvaluator = FakeAgentEvaluator + fake_eval_module.EvalConfig = EvalConfig + fake_eval_module.EvaluationCasesFailed = _EvaluationCasesFailed + monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) + + call_agent_module = types.ModuleType("fake_call_agent_module") + + async def call_agent(query: str) -> str: + return query + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + return calls + + +def _write_sdk_optimizer_config(tmp_path: Path) -> Path: + path = tmp_path / "sdk_optimizer.json" + path.write_text( + json.dumps({ + "evaluate": { + "metrics": [] + }, + "optimize": { + "algorithm": { + "name": "gepa_reflective" + } + }, + }), + encoding="utf-8", + ) + return path + + +def _write_gate_config( + tmp_path: Path, + *, + min_val_score_improvement: float, + max_total_cost: float | None, +) -> Path: + path = tmp_path / "wrapper_gate.json" + path.write_text( + json.dumps( + {"gate": { + "min_val_score_improvement": min_val_score_improvement, + "max_total_cost": max_total_cost, + }}), + encoding="utf-8", + ) + return path diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py new file mode 100644 index 00000000..4b83c957 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_integration.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase +from examples.optimization.eval_optimize_loop.eval_loop.schemas import to_jsonable +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline_async +from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation import AgentOptimizer +from trpc_agent_sdk.evaluation import GepaReflectiveOptimizer +from trpc_agent_sdk.evaluation import TargetPrompt + + +class FakeGEPAResult: + + def __init__( + self, + baseline: dict[str, str], + candidate: dict[str, str], + ) -> None: + self.candidates = [baseline, candidate] + self.val_aggregate_scores = [2 / 3, 1.0] + self.parents = [[None], [0]] + self.discovery_eval_counts = [0, 1] + self.total_metric_calls = 6 + self.best_outputs_valset = None + self.per_objective_best_candidates = {} + + @property + def best_idx(self) -> int: + return 1 + + +@pytest.mark.asyncio +async def test_sdk_pipeline_uses_real_facade_evaluator_and_post_gate_writeback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inputs_dir = tmp_path / "inputs" + inputs_dir.mkdir() + train_path = inputs_dir / "train.evalset.json" + val_path = inputs_dir / "val.evalset.json" + prompt_path = inputs_dir / "system_prompt.txt" + optimizer_config_path = inputs_dir / "optimizer.json" + gate_path = inputs_dir / "gate.json" + shutil.copyfile(DEFAULT_TRAIN, train_path) + shutil.copyfile(DEFAULT_VAL, val_path) + expected_train_case_ids = _eval_ids(train_path) + expected_val_case_ids = _eval_ids(val_path) + shutil.copyfile(DEFAULT_PROMPT, prompt_path) + optimizer_config_path.write_text( + json.dumps( + { + "evaluate": { + "metrics": [{ + "metric_name": "final_response_avg_score", + "threshold": 0.5, + }], + "num_runs": 1, + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": None + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 91, + "reflection_lm": { + "provider_name": "openai", + "model_name": "gpt-4o", + "api_key": "test-key", + }, + "max_metric_calls": 6, + }, + }, + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + gate_path.write_text( + json.dumps( + { + "gate": { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 0.0, + "max_total_cost": None, + } + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + baseline = {"system_prompt": prompt_path.read_text(encoding="utf-8")} + candidate = { + "system_prompt": + (baseline["system_prompt"].rstrip("\n") + "\n\nUse strict JSON only when the user explicitly asks.\n") + } + gepa_calls: list[dict[str, object]] = [] + + async def fake_call_gepa( + self: GepaReflectiveOptimizer, + **kwargs: object, + ) -> FakeGEPAResult: + assert isinstance(self, GepaReflectiveOptimizer) + gepa_calls.append(dict(kwargs)) + return FakeGEPAResult(baseline, candidate) + + async def call_agent(query: str) -> str: + prompt = prompt_path.read_text(encoding="utf-8") + return deterministic_sdk_response(prompt, query) + + monkeypatch.setattr( + GepaReflectiveOptimizer, + "_call_gepa_optimize", + fake_call_gepa, + ) + module = ModuleType("issue91_sdk_call_agent") + module.call_agent = call_agent + monkeypatch.setitem(sys.modules, module.__name__, module) + + report = await run_pipeline_async( + mode="sdk", + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + sdk_call_agent="issue91_sdk_call_agent:call_agent", + gate_config_path=gate_path, + update_source=True, + run_id="sdk-integration", + ) + + assert AgentOptimizer.__module__.startswith("trpc_agent_sdk.evaluation.") + assert AgentEvaluator.__module__.startswith("trpc_agent_sdk.evaluation.") + assert TargetPrompt.__module__.startswith("trpc_agent_sdk.evaluation.") + assert GepaReflectiveOptimizer.__module__.startswith("trpc_agent_sdk.evaluation.") + assert len(gepa_calls) == 1 + gepa_call = gepa_calls[0] + assert gepa_call["seed_candidate"] == baseline + captured_trainset = gepa_call["trainset"] + captured_valset = gepa_call["valset"] + assert isinstance(captured_trainset, list) + assert isinstance(captured_valset, list) + assert {case.eval_id for case in captured_trainset} == expected_train_case_ids + assert {case.eval_id for case in captured_valset} == expected_val_case_ids + assert gepa_call["callbacks"] + assert report.selected_candidate == "sdk_round_001" + assert report.candidates + for record in report.candidates: + assert {case.case_id for case in record["train_result"].cases} == expected_train_case_ids + assert {case.case_id for case in record["validation_result"].cases} == expected_val_case_ids + assert report.gate_decisions + assert all(item.gate_status == "applied" for item in report.gate_decisions) + decision = next(item for item in report.gate_decisions if item.candidate_id == "sdk_round_001") + assert decision.gate_status == "applied" + assert decision.accepted is True + assert report.writeback.status == "applied" + assert prompt_path.read_text(encoding="utf-8") == candidate["system_prompt"] + + sdk_summary = report.audit["sdk_result_summary"] + assert sdk_summary["status"] == "SUCCEEDED" + assert sdk_summary["extras"]["total_metric_calls"] == 6 + assert sdk_summary["total_llm_cost"] == 0.0 + assert report.cost_summary.reported_optimizer_cost == 0.0 + assert "partial_applied" not in json.dumps(to_jsonable(report)) + run_dir = tmp_path / "out" / "runs" / "sdk-integration" + assert (run_dir / "optimizer" / "result.json").is_file() + optimizer_config_snapshot = run_dir / "optimizer" / "config.snapshot.json" + artifact_files = sorted(path for path in run_dir.rglob("*") if path.is_file()) + assert optimizer_config_snapshot in artifact_files + leaked_test_key = [ + path.relative_to(run_dir).as_posix() for path in artifact_files if b"test-key" in path.read_bytes() + ] + assert leaked_test_key == [] + + +def deterministic_sdk_response(prompt: str, query: str) -> str: + case = EvalCase( + case_id="sdk-runtime-query", + split="runtime", + input=query, + expectation={"type": "runtime-only; must not be inspected"}, + ) + output, _, _ = FakeModel(seed=91).generate( + "sdk-runtime", + prompt, + case.input, + ) + try: + parsed = json.loads(output) + except json.JSONDecodeError: + return output + return json.dumps( + parsed, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def _eval_ids(path: Path) -> set[str]: + payload = json.loads(path.read_text(encoding="utf-8")) + return {str(case["eval_id"]) for case in payload["eval_cases"]} diff --git a/examples/optimization/eval_optimize_loop/tests/test_writeback.py b/examples/optimization/eval_optimize_loop/tests/test_writeback.py new file mode 100644 index 00000000..b325334c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_writeback.py @@ -0,0 +1,594 @@ +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop import writeback +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( + ConcurrentPromptUpdateError, ) +from examples.optimization.eval_optimize_loop.eval_loop.writeback import PromptRestorationError +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( + commit_prompt_bundle, ) +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( + snapshot_prompt_files, ) +from examples.optimization.eval_optimize_loop.eval_loop.writeback import ( + temporary_prompt_bundle, ) + + +def _prompt_files(tmp_path: Path) -> tuple[dict[str, Path], dict[str, bytes]]: + paths = { + "system": tmp_path / "system.txt", + "user": tmp_path / "user.txt", + } + baseline = { + "system": b"system baseline\r\n\xff", + "user": "user baseline \u57fa\u7ebf\n".encode(), + } + for name, path in paths.items(): + path.write_bytes(baseline[name]) + return paths, baseline + + +def test_snapshot_prompt_files_preserves_bytes_and_returns_hash_copy(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + + snapshot = snapshot_prompt_files(paths) + + system = snapshot.files["system"] + assert system.name == "system" + assert system.path == paths["system"] + assert system.content == baseline["system"] + assert system.sha256 == hashlib.sha256(baseline["system"]).hexdigest() + + hashes = snapshot.hashes() + hashes["system"] = "mutated copy" + assert snapshot.hashes()["system"] == system.sha256 + + +def test_snapshot_prompt_files_rejects_hardlink_aliases(tmp_path: Path): + prompt_path = tmp_path / "prompt.txt" + alias_path = tmp_path / "prompt-hardlink.txt" + prompt_path.write_text("baseline", encoding="utf-8") + os.link(prompt_path, alias_path) + assert prompt_path.samefile(alias_path) + + with pytest.raises(ValueError, match="different physical files"): + snapshot_prompt_files({"system": prompt_path, "router": alias_path}) + + +def test_temporary_prompt_bundle_restores_exact_bytes_after_body_error(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + candidate = { + "system": "candidate system", + "user": "candidate \u7528\u6237", + } + + with pytest.raises(RuntimeError, match="candidate failed"): + with temporary_prompt_bundle(snapshot, candidate): + assert paths["system"].read_bytes() == b"candidate system" + assert paths["user"].read_bytes() == "candidate \u7528\u6237".encode() + raise RuntimeError("candidate failed") + + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_temporary_prompt_bundle_rejects_missing_prompt_before_writing(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + + with pytest.raises(ValueError, match="user"): + with temporary_prompt_bundle(snapshot, {"system": "candidate"}): + pytest.fail("incomplete prompt bundle must not be entered") + + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_temporary_rejects_stale_snapshot_without_overwriting_external_update(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external before temporary entry" + candidate = { + "system": "candidate system", + "user": "candidate user", + } + paths["user"].write_bytes(external_content) + original_replace = os.replace + replace_calls = 0 + + def count_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "replace", count_replace) + + with pytest.raises(ConcurrentPromptUpdateError, match="changed"): + with temporary_prompt_bundle(snapshot, candidate): + pass + + assert replace_calls == 0 + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_temporary_restore_preserves_external_update_and_reports_conflict(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external during temporary evaluation" + candidate = { + "system": "candidate system", + "user": "candidate user", + } + + with pytest.raises(PromptRestorationError, match="restore"): + with temporary_prompt_bundle(snapshot, candidate): + paths["user"].write_bytes(external_content) + + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_temporary_body_error_remains_primary_when_restore_conflicts(tmp_path: Path): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external before failed body exits" + candidate = { + "system": "candidate system", + "user": "candidate user", + } + + with pytest.raises(RuntimeError, match="candidate failed") as error_info: + with temporary_prompt_bundle(snapshot, candidate): + paths["user"].write_bytes(external_content) + raise RuntimeError("candidate failed") + + diagnostics = " ".join(getattr(error_info.value, "__notes__", [])) + if error_info.value.__cause__ is not None: + diagnostics += f" {error_info.value.__cause__}" + if error_info.value.__context__ is not None: + diagnostics += f" {error_info.value.__context__}" + + assert "restore" in diagnostics + assert "user" in diagnostics + assert isinstance(error_info.value.__cause__, PromptRestorationError) + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_temporary_partial_install_restores_written_file_and_preserves_external_update(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external before second temporary replace" + candidate = { + "system": "candidate system", + "user": "candidate user", + } + original_fsync = os.fsync + fsync_calls = 0 + + def inject_external_before_second_precondition(file_descriptor: int): + nonlocal fsync_calls + original_fsync(file_descriptor) + fsync_calls += 1 + if fsync_calls == 2: + paths["user"].write_bytes(external_content) + + monkeypatch.setattr(writeback.os, "fsync", inject_external_before_second_precondition) + + with pytest.raises(ConcurrentPromptUpdateError, match="changed"): + with temporary_prompt_bundle(snapshot, candidate): + pass + + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_temporary_verifies_complete_candidate_bundle_before_entering_body(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external after second candidate replace" + original_replace = os.replace + replace_calls = 0 + body_entered = False + candidate = { + "system": "candidate system", + "user": "candidate user", + } + + def update_first_file_after_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + original_replace(source, destination) + replace_calls += 1 + if replace_calls == 2: + paths["system"].write_bytes(external_content) + + monkeypatch.setattr(writeback.os, "replace", update_first_file_after_second_replace) + + with pytest.raises((ConcurrentPromptUpdateError, RuntimeError)): + with temporary_prompt_bundle(snapshot, candidate): + body_entered = True + + assert not body_entered + assert paths["system"].read_bytes() == external_content + assert paths["user"].read_bytes() == baseline["user"] + + +def test_commit_rejects_concurrent_update_before_any_write(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"updated by another process" + paths["user"].write_bytes(external_content) + + def unexpected_write(path: Path, content: bytes) -> None: + pytest.fail(f"unexpected write to {path} with {content!r}") + + monkeypatch.setattr(writeback, "_atomic_replace_bytes", unexpected_write) + + with pytest.raises(ConcurrentPromptUpdateError, match="changed"): + commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_commit_rechecks_first_file_after_flush_before_replace(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external between bundle CAS and first replace" + original_fsync = os.fsync + external_injected = False + + def inject_external_before_precondition(file_descriptor: int): + nonlocal external_injected + original_fsync(file_descriptor) + if not external_injected: + paths["system"].write_bytes(external_content) + external_injected = True + + monkeypatch.setattr(writeback.os, "fsync", inject_external_before_precondition) + + with pytest.raises(ConcurrentPromptUpdateError, match="changed"): + commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert external_injected + assert paths["system"].read_bytes() == external_content + assert paths["user"].read_bytes() == baseline["user"] + + +def test_commit_rolls_back_written_file_and_preserves_external_unwritten_file(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external before second file replace" + original_fsync = os.fsync + fsync_calls = 0 + + def inject_external_before_second_precondition(file_descriptor: int): + nonlocal fsync_calls + original_fsync(file_descriptor) + fsync_calls += 1 + if fsync_calls == 2: + paths["user"].write_bytes(external_content) + + monkeypatch.setattr(writeback.os, "fsync", inject_external_before_second_precondition) + + result = commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert result.status == "rollback_failed" + assert result.error is not None + assert "changed" in result.error + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_commit_rejects_final_candidate_hash_mismatch_and_preserves_external_update(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external after final candidate replace" + original_replace = os.replace + replace_calls = 0 + + def update_after_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + original_replace(source, destination) + replace_calls += 1 + if replace_calls == 2: + paths["user"].write_bytes(external_content) + + monkeypatch.setattr(writeback.os, "replace", update_after_second_replace) + + result = commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert result.status == "rollback_failed" + assert result.after_hashes["user"] == hashlib.sha256(external_content).hexdigest() + assert result.error is not None + assert "candidate" in result.error + assert paths["system"].read_bytes() == baseline["system"] + assert paths["user"].read_bytes() == external_content + + +def test_commit_restore_rechecks_candidate_hash_after_flush(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + external_content = b"external between restore checks" + original_fsync = os.fsync + original_replace = os.replace + fsync_calls = 0 + replace_calls = 0 + + def inject_external_during_restore(file_descriptor: int): + nonlocal fsync_calls + original_fsync(file_descriptor) + fsync_calls += 1 + if fsync_calls == 3: + paths["system"].write_bytes(external_content) + + def fail_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 2: + raise OSError("second candidate replace failed") + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "fsync", inject_external_during_restore) + monkeypatch.setattr(writeback.os, "replace", fail_second_replace) + + result = commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert result.status == "rollback_failed" + assert result.error is not None + assert "changed" in result.error + assert paths["system"].read_bytes() == external_content + assert paths["user"].read_bytes() == baseline["user"] + + +def test_commit_rejects_missing_prompt_before_writing(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + + def unexpected_write(path: Path, content: bytes) -> None: + pytest.fail(f"unexpected write to {path} with {content!r}") + + monkeypatch.setattr(writeback, "_atomic_replace_bytes", unexpected_write) + + with pytest.raises(ValueError, match="user"): + commit_prompt_bundle(snapshot, {"system": "candidate"}) + + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_commit_applies_complete_bundle_and_reports_hashes(tmp_path: Path): + paths, _ = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + prompts = {"system": "new system", "user": "new \u7528\u6237"} + + result = commit_prompt_bundle(snapshot, prompts) + + assert result.status == "applied" + assert result.before_hashes == snapshot.hashes() + assert result.after_hashes == {name: hashlib.sha256(prompts[name].encode()).hexdigest() for name in paths} + assert result.error is None + assert {name: path.read_bytes() for name, path in paths.items()} == {name: prompts[name].encode() for name in paths} + + +def test_commit_rolls_back_when_post_write_hashing_fails(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + original_read_bytes = Path.read_bytes + hash_read_failed = False + + def fail_once_for_candidate_system(path: Path): + nonlocal hash_read_failed + content = original_read_bytes(path) + if not hash_read_failed and path == paths["system"] and content == b"candidate system": + hash_read_failed = True + raise OSError("post-write hash read failed") + return content + + monkeypatch.setattr(Path, "read_bytes", fail_once_for_candidate_system) + + result = commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert hash_read_failed + assert result.status == "rolled_back" + assert result.before_hashes == snapshot.hashes() + assert result.after_hashes == snapshot.hashes() + assert result.error is not None + assert "post-write hash read failed" in result.error + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_commit_removes_temp_file_when_replace_fails(tmp_path: Path, monkeypatch): + target = tmp_path / "system.txt" + baseline = b"system baseline" + target.write_bytes(baseline) + snapshot = snapshot_prompt_files({"system": target}) + original_replace = os.replace + replace_calls = 0 + + def fail_first_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 1: + raise OSError("candidate replace failed") + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "replace", fail_first_replace) + + result = commit_prompt_bundle(snapshot, {"system": "candidate system"}) + + assert result.status == "rolled_back" + assert target.read_bytes() == baseline + assert list(tmp_path.glob(f".{target.name}.*.tmp")) == [] + + +def test_atomic_replace_preserves_replace_error_when_temp_unlink_fails(tmp_path: Path, monkeypatch): + target = tmp_path / "system.txt" + baseline = b"system baseline" + target.write_bytes(baseline) + original_unlink = Path.unlink + + def fail_replace(source: str | bytes | Path, destination: str | bytes | Path): + raise OSError("primary replace failed") + + def fail_temp_unlink(path: Path, missing_ok: bool = False): + if path.parent == tmp_path and path.name.startswith(f".{target.name}.") and path.suffix == ".tmp": + raise OSError("temp unlink failed") + return original_unlink(path, missing_ok=missing_ok) + + with monkeypatch.context() as injected_failure: + injected_failure.setattr(writeback.os, "replace", fail_replace) + injected_failure.setattr(Path, "unlink", fail_temp_unlink) + with pytest.raises(OSError) as error_info: + writeback._atomic_replace_bytes(target, b"candidate system") + + for temp_path in tmp_path.glob(f".{target.name}.*.tmp"): + original_unlink(temp_path) + + assert str(error_info.value) == "primary replace failed" + assert target.read_bytes() == baseline + + +def test_commit_rolls_back_written_file_when_second_replace_fails(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + original_replace = os.replace + replace_calls = 0 + + def fail_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 2: + raise OSError("second replace failed") + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "replace", fail_second_replace) + + result = commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert result.status == "rolled_back" + assert result.before_hashes == snapshot.hashes() + assert result.after_hashes == snapshot.hashes() + assert result.error is not None + assert "second replace failed" in result.error + assert {name: path.read_bytes() for name, path in paths.items()} == baseline + + +def test_commit_reports_rollback_failed_when_restored_hashes_differ(tmp_path: Path, monkeypatch): + paths, _ = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + original_replace = os.replace + original_restore = writeback._restore_snapshot + replace_calls = 0 + + def fail_second_replace(source: str | bytes | Path, destination: str | bytes | Path): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 2: + raise OSError("candidate write failed") + return original_replace(source, destination) + + def restore_then_concurrently_update(snapshot_to_restore, written_hashes): + failures = original_restore(snapshot_to_restore, written_hashes) + paths["system"].write_bytes(b"concurrent update after restore") + return failures + + monkeypatch.setattr(writeback.os, "replace", fail_second_replace) + monkeypatch.setattr(writeback, "_restore_snapshot", restore_then_concurrently_update) + + result = commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert result.status == "rollback_failed" + assert result.after_hashes["system"] != snapshot.hashes()["system"] + assert result.error is not None + assert "system: final hash differs from snapshot" in result.error + + +def test_commit_reports_rollback_failed_when_written_file_restore_fails(tmp_path: Path, monkeypatch): + paths, baseline = _prompt_files(tmp_path) + snapshot = snapshot_prompt_files(paths) + original_replace = os.replace + replace_calls = 0 + + def fail_write_and_first_restore( + source: str | bytes | Path, + destination: str | bytes | Path, + ): + nonlocal replace_calls + replace_calls += 1 + if replace_calls == 2: + raise OSError("candidate write failed") + if replace_calls == 3: + raise OSError("system restore failed") + return original_replace(source, destination) + + monkeypatch.setattr(writeback.os, "replace", fail_write_and_first_restore) + + result = commit_prompt_bundle( + snapshot, + { + "system": "candidate system", + "user": "candidate user" + }, + ) + + assert result.status == "rollback_failed" + assert result.before_hashes == snapshot.hashes() + assert result.after_hashes["system"] != snapshot.hashes()["system"] + assert result.after_hashes["user"] == snapshot.hashes()["user"] + assert result.error is not None + assert "candidate write failed" in result.error + assert "system restore failed" in result.error + assert paths["system"].read_bytes() == b"candidate system" + assert paths["user"].read_bytes() == baseline["user"] diff --git a/pyproject.toml b/pyproject.toml index 6c47d4d5..146e2281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,5 +204,5 @@ split_before_logical_operator = true [tool.pytest.ini_options] minversion = "6.0" addopts = "-ra -q" -testpaths = ["tests"] +testpaths = ["tests", "examples/optimization/eval_optimize_loop/tests"] asyncio_mode = "auto" diff --git a/tests/evaluation/test_agent_evaluator.py b/tests/evaluation/test_agent_evaluator.py index 8609ccc5..81324149 100644 --- a/tests/evaluation/test_agent_evaluator.py +++ b/tests/evaluation/test_agent_evaluator.py @@ -5,10 +5,13 @@ # tRPC-Agent-Python is licensed under Apache-2.0. """Unit tests for agent evaluator (agent_evaluator).""" +import json + import pytest import trpc_agent_sdk.runners # noqa: F401 +from trpc_agent_sdk.evaluation import _agent_evaluator as agent_evaluator_module from trpc_agent_sdk.evaluation import EvalStatus from trpc_agent_sdk.evaluation import EvalCaseResult from trpc_agent_sdk.evaluation import EvalMetricResult @@ -18,6 +21,68 @@ from trpc_agent_sdk.evaluation import PassNC +def test_evaluation_cases_failed_is_public_with_private_alias(): + from trpc_agent_sdk import evaluation + from trpc_agent_sdk.evaluation._agent_evaluator import _EvaluationCasesFailed + + assert evaluation.EvaluationCasesFailed is _EvaluationCasesFailed + assert issubclass(evaluation.EvaluationCasesFailed, AssertionError) + + +@pytest.mark.parametrize( + ("raw_path", "expected"), + [ + (r"C:\x\a.evalset.json", (r"C:\x\a.evalset.json", None)), + ( + r"C:\x\a.evalset.json:case-1", + (r"C:\x\a.evalset.json", "case-1"), + ), + ( + "/tmp/a.evalset.json:case-1", + ("/tmp/a.evalset.json", "case-1"), + ), + ], +) +def test_split_eval_set_selector_is_drive_safe(raw_path, expected): + assert agent_evaluator_module._split_eval_set_selector(raw_path) == expected + + +def test_load_eval_set_from_absolute_path_and_selector(tmp_path): + evalset_path = (tmp_path / "sample.evalset.json").resolve() + evalset_path.write_text( + json.dumps({ + "eval_set_id": "set", + "eval_cases": [{ + "eval_id": "case-1", + "conversation": [{ + "invocation_id": "turn-1", + "user_content": { + "role": "user", + "parts": [{"text": "query"}], + }, + "final_response": { + "role": "model", + "parts": [{"text": "expected"}], + }, + }], + }], + }), + encoding="utf-8", + ) + + loaded = AgentEvaluator._load_eval_set_from_file( + str(evalset_path), + agent_evaluator_module.EvalConfig(criteria={}), + ) + selected = AgentEvaluator._load_eval_set_from_file( + f"{evalset_path}:case-1", + agent_evaluator_module.EvalConfig(criteria={}), + ) + + assert [case.eval_id for case in loaded.eval_cases] == ["case-1"] + assert [case.eval_id for case in selected.eval_cases] == ["case-1"] + + class TestPassNC: """Test suite for PassNC dataclass.""" diff --git a/tests/evaluation/test_agent_optimizer.py b/tests/evaluation/test_agent_optimizer.py index d7b061f7..6391d8e9 100644 --- a/tests/evaluation/test_agent_optimizer.py +++ b/tests/evaluation/test_agent_optimizer.py @@ -254,6 +254,8 @@ async def test_facade_persists_artifacts_under_output_dir(tmp_path, monkeypatch) """The facade must materialise result.json, summary.txt, rounds/*.json, baseline_prompts/, best_prompts/, config.snapshot.json and run.log under output_dir for every successful run.""" + import json + train = EvalSet(eval_set_id="train", eval_cases=[_eval_case("c1")]) val = EvalSet(eval_set_id="val", eval_cases=[_eval_case("c1")]) train_path = tmp_path / "train.json" @@ -287,7 +289,14 @@ async def fake_call_gepa(self, **kwargs): assert (output_dir / "result.json").is_file() assert (output_dir / "summary.txt").is_file() - assert (output_dir / "config.snapshot.json").is_file() + config_snapshot_path = output_dir / "config.snapshot.json" + assert config_snapshot_path.is_file() + config_snapshot_text = config_snapshot_path.read_text(encoding="utf-8") + assert "test-key" not in config_snapshot_text + config_snapshot = json.loads(config_snapshot_text) + assert config_snapshot["optimize"]["algorithm"]["reflection_lm"][ + "api_key" + ] == "" assert (output_dir / "run.log").is_file() assert (output_dir / "baseline_prompts" / "instruction.md").is_file() assert (output_dir / "best_prompts" / "instruction.md").is_file() @@ -297,6 +306,114 @@ async def fake_call_gepa(self, **kwargs): assert "SUCCEEDED" in log_line +def test_copy_config_snapshot_recursively_redacts_common_secret_keys(tmp_path): + """Config snapshots must remain useful without publishing credentials.""" + import json + + payload = { + "api_key": "api-secret", + "nested": [ + { + "TOKEN": "token-secret", + "access-token": "access-secret", + "Authorization": "Bearer auth-secret", + }, + { + "password": "password-secret", + "credentials": {"username": "alice", "password": "nested-secret"}, + "privateKey": "private-secret", + }, + { + "openai_api_key": "namespaced-api-secret", + "github_token": "namespaced-token-secret", + "aws_secret_access_key": "aws-secret", + "db_passwd": "database-secret", + }, + ], + "model_name": "gpt-4o", + "max_tokens": 128, + "token_budget": 256, + } + config_path = tmp_path / "optimizer.json" + config_path.write_text(json.dumps(payload), encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + + AgentOptimizer._copy_config_snapshot(str(config_path), str(output_dir)) + + snapshot_path = output_dir / "config.snapshot.json" + snapshot_text = snapshot_path.read_text(encoding="utf-8") + snapshot = json.loads(snapshot_text) + assert snapshot == { + "api_key": "", + "nested": [ + { + "TOKEN": "", + "access-token": "", + "Authorization": "", + }, + { + "password": "", + "credentials": "", + "privateKey": "", + }, + { + "openai_api_key": "", + "github_token": "", + "aws_secret_access_key": "", + "db_passwd": "", + }, + ], + "model_name": "gpt-4o", + "max_tokens": 128, + "token_budget": 256, + } + assert snapshot_text == ( + json.dumps( + snapshot, + ensure_ascii=False, + sort_keys=True, + indent=2, + allow_nan=False, + ) + + "\n" + ) + for secret in ( + "api-secret", + "token-secret", + "access-secret", + "auth-secret", + "password-secret", + "nested-secret", + "private-secret", + "namespaced-api-secret", + "namespaced-token-secret", + "aws-secret", + "database-secret", + ): + assert secret not in snapshot_text + + +@pytest.mark.parametrize( + "invalid_config", + [ + '{"api_key": "secret", invalid}', + '{"api_key": NaN}', + ], +) +def test_copy_config_snapshot_invalid_json_fails_closed(tmp_path, invalid_config): + """Malformed source config must never be copied verbatim as an artifact.""" + config_path = tmp_path / "optimizer.json" + config_path.write_text(invalid_config, encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + + AgentOptimizer._copy_config_snapshot(str(config_path), str(output_dir)) + + assert not (output_dir / "config.snapshot.json").exists() + assert not (output_dir / "config.snapshot.json.tmp").exists() + + @pytest.mark.asyncio async def test_facade_persists_artifacts_when_algorithm_fails(tmp_path, monkeypatch): """Even when the algorithm returns a FAILED result the facade should diff --git a/tests/evaluation/test_target_prompt.py b/tests/evaluation/test_target_prompt.py index 8a2c7b48..6858e2e0 100644 --- a/tests/evaluation/test_target_prompt.py +++ b/tests/evaluation/test_target_prompt.py @@ -129,6 +129,30 @@ async def test_read_all_with_paths(tmp_path: Path): assert await target.read_all() == {"a": "alpha", "b": "beta"} +@pytest.mark.parametrize( + "original_bytes", + [ + "system prompt\nkeep exact bytes\n".encode("utf-8"), + "system prompt\r\nkeep exact bytes\r\n".encode("utf-8"), + "lf\ncrlf\r\nlone carriage return\rend".encode("utf-8"), + ], + ids=["lf", "crlf", "mixed"], +) +@pytest.mark.asyncio +async def test_path_read_write_round_trip_preserves_newline_bytes( + tmp_path: Path, + original_bytes: bytes, +): + prompt_path = tmp_path / "prompt.md" + prompt_path.write_bytes(original_bytes) + target = TargetPrompt().add_path("prompt", str(prompt_path)) + + snapshot = await target.read_all() + await target.write_all(snapshot) + + assert prompt_path.read_bytes() == original_bytes + + @pytest.mark.asyncio async def test_read_all_path_not_exist_raises_file_not_found(tmp_path: Path): target = TargetPrompt().add_path("missing", str(tmp_path / "ghost.md")) diff --git a/trpc_agent_sdk/evaluation/__init__.py b/trpc_agent_sdk/evaluation/__init__.py index 8f614b4b..b70ac64a 100644 --- a/trpc_agent_sdk/evaluation/__init__.py +++ b/trpc_agent_sdk/evaluation/__init__.py @@ -34,6 +34,7 @@ # Main evaluation entry point from ._agent_evaluator import AgentEvaluator +from ._agent_evaluator import EvaluationCasesFailed from ._agent_evaluator import PassNC from ._common import EvalBaseModel from ._criterion_registry import CRITERION_REGISTRY @@ -220,6 +221,7 @@ "DEFAULT_OPTIMIZE_MAX_TOKENS", "DEFAULT_OPTIMIZE_TEMPERATURE", "EvaluationOutcome", + "EvaluationCasesFailed", "FinishReason", "FrameworkStopConfig", "GepaReflectiveAlgo", diff --git a/trpc_agent_sdk/evaluation/_agent_evaluator.py b/trpc_agent_sdk/evaluation/_agent_evaluator.py index a2e47009..8f5a0641 100644 --- a/trpc_agent_sdk/evaluation/_agent_evaluator.py +++ b/trpc_agent_sdk/evaluation/_agent_evaluator.py @@ -75,7 +75,7 @@ _RESULT_HANDLER = _utils.EvalResultHandler() -class _EvaluationCasesFailed(AssertionError): +class EvaluationCasesFailed(AssertionError): """Signal raised by ``_EvalExecuter._run`` when one or more eval cases fail. Subclasses :class:`AssertionError` so direct ``AgentEvaluator.evaluate`` @@ -93,6 +93,19 @@ class _EvaluationCasesFailed(AssertionError): """ +_EvaluationCasesFailed = EvaluationCasesFailed + + +def _split_eval_set_selector(eval_set_file: str) -> tuple[str, Optional[str]]: + """Split an optional .json:case_id suffix without consuming a drive.""" + + drive, tail = os.path.splitdrive(eval_set_file) + json_path, separator, case_id = tail.rpartition(":") + if not separator or not case_id or not json_path.lower().endswith(".json"): + return (eval_set_file, None) + return (drive + json_path, case_id) + + @dataclass(frozen=True) class PassNC: """(n, c): n = runs, c = runs that all passed (for pass@k / pass^k).""" @@ -660,14 +673,7 @@ def _load_eval_set_from_file( FileNotFoundError: If file doesn't exist ValueError: If file format is invalid or eval case not found """ - # Check if file_path contains a case selector (ADK style: "file.json:case_id") - selected_case_id = None - actual_file_path = eval_set_file - - if ":" in eval_set_file: - parts = eval_set_file.split(":", 1) - actual_file_path = parts[0] - selected_case_id = parts[1] + actual_file_path, selected_case_id = _split_eval_set_selector(eval_set_file) if not os.path.exists(actual_file_path): raise FileNotFoundError(f"Eval set file not found: {actual_file_path}") diff --git a/trpc_agent_sdk/evaluation/_agent_optimizer.py b/trpc_agent_sdk/evaluation/_agent_optimizer.py index 16510e75..76641655 100644 --- a/trpc_agent_sdk/evaluation/_agent_optimizer.py +++ b/trpc_agent_sdk/evaluation/_agent_optimizer.py @@ -15,6 +15,7 @@ from __future__ import annotations import inspect +import json import logging import os import signal @@ -52,6 +53,54 @@ _PROMPT_FILE_LOGGER = logging.getLogger("trpc_agent_sdk.optimizer") +_REDACTED_CONFIG_VALUE = "" +_SENSITIVE_CONFIG_KEY_SUFFIXES = ( + "apikey", + "authorization", + "authtoken", + "bearertoken", + "clientsecret", + "credential", + "credentials", + "idtoken", + "password", + "passwd", + "privatekey", + "refreshtoken", + "secret", + "secretaccesskey", + "secretkey", + "token", + "accesstoken", +) + + +def _reject_non_finite_json_constant(value: str) -> Any: + """Reject Python's non-standard JSON constants (NaN and infinities).""" + raise ValueError(f"non-finite JSON constant is not allowed: {value}") + + +def _redact_config_secrets(value: Any) -> Any: + """Return a JSON-compatible copy with credential values redacted. + + Key matching is case-insensitive, ignores separators and accepts + namespaced credential suffixes, so ``api_key``, ``API-Key`` and + ``openai_api_key`` share the same policy without redacting safe fields + such as ``max_tokens``. + """ + if isinstance(value, dict): + redacted = {} + for key, child in value.items(): + normalized_key = "".join(character for character in key.casefold() if character.isalnum()) + if normalized_key.endswith(_SENSITIVE_CONFIG_KEY_SUFFIXES): + redacted[key] = _REDACTED_CONFIG_VALUE + else: + redacted[key] = _redact_config_secrets(child) + return redacted + if isinstance(value, list): + return [_redact_config_secrets(child) for child in value] + return value + def _atomic_write_text(path: str, content: str) -> None: """Atomically replace ``path`` with ``content`` (UTF-8). @@ -157,8 +206,8 @@ async def optimize( output_dir: Required artifact directory. The facade creates it when missing and persists ``result.json``, ``summary.txt``, ``rounds/`` records, ``baseline_prompts/`` and ``best_prompts/`` - directories, a ``config.snapshot.json`` copy of the input - config, and a ``run.log`` summary line. + directories, a redacted ``config.snapshot.json`` copy of the + input config, and a ``run.log`` summary line. callbacks: Optional evaluator lifecycle callbacks. update_source: When True, persist the best candidate back to every registered TargetPrompt field after a SUCCEEDED @@ -393,7 +442,7 @@ def _persist_artifacts( (regardless of update_source). - ``best_prompts/.md`` Best candidate per field (only when a result was produced). - - ``config.snapshot.json`` Copy of the input config. + - ``config.snapshot.json`` Redacted copy of the input config. - ``run.log`` One-line status footer. SIGINT (Ctrl+C) is masked for the duration of this method so a @@ -473,12 +522,25 @@ def _write_rounds_directory(result: OptimizeResult, output_dir: str) -> None: def _copy_config_snapshot(config_path: str, output_dir: str) -> None: target = os.path.join(output_dir, "config.snapshot.json") try: - # Read + atomic-write rather than shutil.copyfile so an interrupted - # copy cannot leave a half-written ``config.snapshot.json``. - content = Path(config_path).read_text(encoding="utf-8") + # Parse and redact fully before the first write so neither the + # destination nor its temporary sibling can ever contain the raw + # credential-bearing config. Invalid/non-finite JSON therefore + # fails closed instead of falling back to a plaintext copy. + raw_config = json.loads( + Path(config_path).read_text(encoding="utf-8"), + parse_constant=_reject_non_finite_json_constant, + ) + redacted_config = _redact_config_secrets(raw_config) + content = json.dumps( + redacted_config, + ensure_ascii=False, + sort_keys=True, + indent=2, + allow_nan=False, + ) + "\n" _atomic_write_text(target, content) except Exception: # pragma: no cover - _PROMPT_FILE_LOGGER.warning("failed to copy config snapshot", exc_info=True) + _PROMPT_FILE_LOGGER.warning("failed to write redacted config snapshot", exc_info=True) @staticmethod def _write_run_log(*, output_dir: str, line: str) -> None: diff --git a/trpc_agent_sdk/evaluation/_target_prompt.py b/trpc_agent_sdk/evaluation/_target_prompt.py index da104f15..7fb6fd03 100644 --- a/trpc_agent_sdk/evaluation/_target_prompt.py +++ b/trpc_agent_sdk/evaluation/_target_prompt.py @@ -180,7 +180,7 @@ def _snapshot_path_contents(self) -> dict[str, Optional[str]]: for name, src in self._sources.items(): if isinstance(src, _PathSource): try: - snapshot[name] = Path(src.path).read_text(encoding="utf-8") + snapshot[name] = self._read_path(src.path) except FileNotFoundError: snapshot[name] = None return snapshot @@ -232,12 +232,18 @@ def _cleanup_tmp_files(self) -> None: @staticmethod def _atomic_write_path(path: str, content: str) -> None: tmp = path + ".tmp" - Path(tmp).write_text(content, encoding="utf-8") + with Path(tmp).open("w", encoding="utf-8", newline="") as prompt_file: + prompt_file.write(content) os.replace(tmp, path) + @staticmethod + def _read_path(path: str) -> str: + with Path(path).open("r", encoding="utf-8", newline="") as prompt_file: + return prompt_file.read() + async def _read_one(self, src: _Source) -> str: if isinstance(src, _PathSource): - return Path(src.path).read_text(encoding="utf-8") + return self._read_path(src.path) if isinstance(src, _CallbackSource): return await src.read_fn() raise TypeError(f"unknown source type: {type(src).__name__}")