fix(examples): isolate fake model training evidence

Author: YAO-001

examples/optimization/eval_optimize_loop/eval_loop/evaluator.py

@@ -29,7 +29,7 @@ def __init__(self, model: FakeModel, judge: FakeJudge, *, trace_enabled: bool =
     def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult:
         case_results: list[CaseResult] = []
         for case in cases:
-            output, model_trace, cost = self.model.generate(prompt_id, prompt, case)
+            output, model_trace, cost = self.model.generate(prompt_id, prompt, case.input)
             judged = self.judge.score(case, output)
             failure_category = None
             failure_reason = None

examples/optimization/eval_optimize_loop/eval_loop/fake_model.py

@@ -7,8 +7,6 @@
 from dataclasses import dataclass
 from typing import Any
 
-from .schemas import EvalCase
-
 _ASSIGNMENT_PATTERN = re.compile(r"(?<![A-Za-z0-9_-])([A-Za-z_][A-Za-z0-9_]*)=([A-Za-z0-9_-]+)" r"(?![A-Za-z0-9_-])")
 _RETURN_ONLY_PATTERN = re.compile(
     r"\breturn\s+only\s+([A-Za-z0-9_-]+)\b",
@@ -40,10 +38,13 @@ def generate(
         self,
         prompt_id: str,
         prompt: str,
-        case: EvalCase,
+        user_input: str,
     ) -> tuple[str, dict[str, Any], float]:
+        if not isinstance(user_input, str):
+            raise TypeError("user_input must be a string")
+
         mode = self._mode(prompt)
-        request = self._parse_request(case.input)
+        request = self._parse_request(user_input)
         output = self._render(request, mode=mode)
         trace = {
             "seed": self.seed,

examples/optimization/eval_optimize_loop/eval_loop/optimizer.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+from collections import Counter
+
 from .diffing import make_unified_diff
 from .schemas import CandidatePrompt
 from .schemas import EvalResult
@@ -23,18 +25,30 @@ def propose(
         baseline_train: EvalResult,
         failure_summary: dict[str, object],
     ) -> list[CandidatePrompt]:
-        failed_cases = [case for case in baseline_train.cases if not case.passed]
-        if not failed_cases:
-            return []
+        if not isinstance(failure_summary, dict):
+            raise TypeError("failure_summary must be a dict")
 
-        observed_categories = {case.failure_category for case in failed_cases if case.failure_category}
-        by_category = failure_summary.get("by_category")
-        if isinstance(by_category, dict):
-            observed_categories.update(
-                str(category) for category, count in by_category.items() if _is_positive_count(count)
+        if baseline_train.split != "train":
+            raise ValueError(
+                "baseline_train.split must be 'train'; "
+                f"got {baseline_train.split!r}"
             )
+        for case in baseline_train.cases:
+            if case.split != "train":
+                raise ValueError(
+                    f"baseline_train case {case.case_id!r} split must be 'train'; "
+                    f"got {case.split!r}"
+                )
 
-        targeted = sorted(observed_categories & _TARGET_FAILURE_CATEGORIES)
+        failed_cases = [case for case in baseline_train.cases if not case.passed]
+        observed_counts = Counter(
+            case.failure_category
+            for case in failed_cases
+            if case.failure_category
+        )
+        _validate_failure_summary(failure_summary, observed_counts)
+
+        targeted = sorted(observed_counts.keys() & _TARGET_FAILURE_CATEGORIES)
         if not targeted:
             return []
 
@@ -75,5 +89,34 @@ def propose(
         ]
 
 
-def _is_positive_count(value: object) -> bool:
-    return not isinstance(value, bool) and isinstance(value, (int, float)) and value > 0
+def _validate_failure_summary(
+    failure_summary: dict[str, object],
+    observed_counts: Counter[str],
+) -> None:
+    if "by_category" not in failure_summary:
+        return
+
+    by_category = failure_summary["by_category"]
+    if not isinstance(by_category, dict):
+        raise ValueError("failure_summary['by_category'] must be a dict")
+
+    summary_counts: dict[object, int] = {}
+    for category, count in by_category.items():
+        summary_counts[category] = _normalize_positive_count(category, count)
+
+    if summary_counts != dict(observed_counts):
+        raise ValueError(
+            "failure_summary['by_category'] must match failed train cases exactly; "
+            f"summary={summary_counts!r}, observed={dict(observed_counts)!r}"
+        )
+
+
+def _normalize_positive_count(category: object, value: object) -> int:
+    normalized = value if type(value) is int and value > 0 else None
+
+    if normalized is None:
+        raise ValueError(
+            "failure_summary['by_category'] count must be a positive integer; "
+            f"category={category!r}, count={value!r}"
+        )
+    return normalized