fix(examples): finalize report timing semantics

Author: YAO-001

examples/optimization/eval_optimize_loop/eval_loop/backends.py

@@ -5,6 +5,7 @@
 import asyncio
 import importlib
 import math
+import time
 from collections.abc import Iterable
 from dataclasses import dataclass
 from dataclasses import replace
@@ -130,13 +131,21 @@ async def optimize_candidates(
             baseline_prompts,
             context="cannot optimize fake prompt bundle",
         )
+        proposal_started_at = time.perf_counter()
         candidates = _normalize_fake_candidates(
             self._optimizer.propose(
                 baseline_prompt,
                 baseline_train,
                 failure_summary,
             )
         )
+        proposal_duration_seconds = _positive_perf_duration(
+            proposal_started_at,
+            time.perf_counter(),
+        )
+        round_duration_seconds = (
+            proposal_duration_seconds / len(candidates) if candidates else 0.0
+        )
         zero_cost = CostSummary(complete=True)
         rounds = [
             OptimizationRound(
@@ -146,7 +155,7 @@ async def optimize_candidates(
                 rationale=candidate.rationale,
                 metrics={},
                 cost=zero_cost,
-                duration_seconds=0.0,
+                duration_seconds=round_duration_seconds,
             )
             for index, candidate in enumerate(candidates, start=1)
         ]
@@ -158,6 +167,10 @@ async def optimize_candidates(
                 "backend": "fake",
                 "baseline_prompt_id": baseline_train.prompt_id,
                 "failure_summary": _safe_jsonable(failure_summary),
+                "proposal_duration_seconds": proposal_duration_seconds,
+                "round_duration_allocation": (
+                    "equal_share_of_batch_proposal_duration"
+                ),
             },
         )
 
@@ -837,6 +850,19 @@ def _finite_number(value: Any, *, context: str) -> float:
     return number
 
 
+def _positive_perf_duration(started_at: float, finished_at: float) -> float:
+    """Return one finite positive duration for a measured perf-counter span."""
+
+    elapsed = finished_at - started_at
+    if math.isfinite(elapsed) and elapsed > 0.0:
+        return elapsed
+
+    resolution = float(time.get_clock_info("perf_counter").resolution)
+    if not math.isfinite(resolution) or resolution <= 0.0:
+        raise RuntimeError("perf_counter resolution must be finite and positive")
+    return resolution
+
+
 def _nonnegative_number(value: Any, *, context: str) -> float:
     number = _finite_number(value, context=context)
     if number < 0.0:

examples/optimization/eval_optimize_loop/eval_loop/report.py

@@ -152,7 +152,7 @@ def build_report(
             if isinstance(result, EvalResult):
                 all_results.append(result)
     return OptimizationReport(
-        schema_version="eval_optimize_loop.v1",
+        schema_version="eval_optimize_loop.v2",
         run=run,
         baseline={"train": baseline_train, "validation": baseline_validation},
         baseline_train=baseline_train,
@@ -645,7 +645,7 @@ def render_markdown(report: OptimizationReport) -> str:
         "",
         "## Failure Attribution Summary",
         "",
-            f"Total failed case evaluations: {summary['total_failed_cases']}",
+        f"Total failed case evaluations: {summary['total_failed_cases']}",
         "",
         "| category | count |",
         "| --- | ---: |",
@@ -691,8 +691,8 @@ def render_markdown(report: OptimizationReport) -> str:
         candidate = record["candidate"]
         lines.extend([
             f"### {candidate.candidate_id}",
-                "",
-                "```diff",
+            "",
+            "```diff",
             candidate.prompt_diff,
             "```",
             "",
@@ -702,8 +702,8 @@ def render_markdown(report: OptimizationReport) -> str:
         "## Reproducibility",
         "",
         f"```{report.run.get('reproducibility_shell') or 'bash'}",
-            report.run.get("reproducibility_command")
-            or report.audit.get("reproducibility_command")
+        report.run.get("reproducibility_command")
+        or report.audit.get("reproducibility_command")
         or REPRODUCIBILITY_COMMAND,
         "```",
         "",

examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py

@@ -6,7 +6,6 @@
 import sys
 
 from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG
-from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT
 from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN
 from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL
 from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline
@@ -15,11 +14,18 @@
 
 def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path):
     output_dir = tmp_path / "run"
+    prompt_path = tmp_path / "baseline_system_prompt.txt"
+    prompt_path.write_text(
+        "You are a helpful support assistant.\n\n"
+        "Answer clearly and include a short explanation when it may help the user.\n"
+        "If the user asks for structured data, provide the information they need.\n",
+        encoding="utf-8",
+    )
     report = run_pipeline(
         train_path=DEFAULT_TRAIN,
         val_path=DEFAULT_VAL,
         optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG,
-        prompt_path=DEFAULT_PROMPT,
+        prompt_path=prompt_path,
         output_dir=output_dir,
         fake_model=True,
         fake_judge=True,
@@ -31,8 +37,10 @@ def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path):
     assert json_path.is_file()
     assert md_path.is_file()
     assert report.selected_candidate == "candidate_002_safe"
+    assert report.schema_version == "eval_optimize_loop.v2"
 
     payload = json.loads(json_path.read_text(encoding="utf-8"))
+    assert payload["schema_version"] == "eval_optimize_loop.v2"
     assert set(payload) >= {
         "schema_version",
         "run",
@@ -213,5 +221,11 @@ def _normalized_payload(payload: dict) -> dict:
     normalized["run"].pop("reproducibility_command", None)
     normalized["audit"].pop("duration_seconds", None)
     normalized["audit"].pop("reproducibility_command", None)
+    normalized["audit"].get("sdk_result_summary", {}).pop(
+        "proposal_duration_seconds",
+        None,
+    )
     normalized["audit"].get("writeback_journal", {}).pop("run_id", None)
+    for round_record in normalized.get("rounds", []):
+        round_record.pop("duration_seconds", None)
     return normalized

examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py

@@ -3,6 +3,7 @@
 import builtins
 import inspect
 import json
+import math
 import os
 import shlex
 import sys
@@ -127,6 +128,75 @@ async def test_fake_backend_wraps_candidates_in_complete_optimization_result(tmp
     assert all(candidate.bundle() == {"system_prompt": candidate.prompt} for candidate in result.candidates)
 
 
+@pytest.mark.asyncio
+async def test_fake_backend_distributes_measured_batch_proposal_duration(
+    tmp_path: Path,
+    monkeypatch,
+):
+    perf_counter_values = iter((10.0, 10.006))
+    monkeypatch.setattr(
+        backend_module,
+        "time",
+        types.SimpleNamespace(
+            perf_counter=lambda: next(perf_counter_values),
+            get_clock_info=lambda name: types.SimpleNamespace(resolution=1e-9),
+        ),
+        raising=False,
+    )
+
+    result = await backend_module.FakeBackend(seed=91).optimize_candidates(
+        baseline_prompts={"system_prompt": "baseline prompt\n"},
+        baseline_train=_failed_fake_train_result(),
+        failure_summary={"by_category": {"format_violation": 1}},
+        train_path=tmp_path / "train.evalset.json",
+        validation_path=tmp_path / "validation.evalset.json",
+        config_path=tmp_path / "optimizer.json",
+        artifact_dir=tmp_path / "fake_optimize",
+    )
+
+    durations = [round_record.duration_seconds for round_record in result.rounds]
+    assert durations == pytest.approx([0.003, 0.003])
+    assert all(duration > 0 and math.isfinite(duration) for duration in durations)
+    assert sum(durations) == pytest.approx(0.006)
+    assert result.raw_summary["proposal_duration_seconds"] == pytest.approx(0.006)
+    assert result.raw_summary["round_duration_allocation"] == (
+        "equal_share_of_batch_proposal_duration"
+    )
+
+
+@pytest.mark.asyncio
+async def test_fake_backend_uses_clock_resolution_when_batch_timer_does_not_advance(
+    tmp_path: Path,
+    monkeypatch,
+):
+    perf_counter_values = iter((42.0, 42.0))
+    monkeypatch.setattr(
+        backend_module,
+        "time",
+        types.SimpleNamespace(
+            perf_counter=lambda: next(perf_counter_values),
+            get_clock_info=lambda name: types.SimpleNamespace(resolution=1e-6),
+        ),
+        raising=False,
+    )
+
+    result = await backend_module.FakeBackend(seed=91).optimize_candidates(
+        baseline_prompts={"system_prompt": "baseline prompt\n"},
+        baseline_train=_failed_fake_train_result(),
+        failure_summary={"by_category": {"format_violation": 1}},
+        train_path=tmp_path / "train.evalset.json",
+        validation_path=tmp_path / "validation.evalset.json",
+        config_path=tmp_path / "optimizer.json",
+        artifact_dir=tmp_path / "fake_optimize",
+    )
+
+    durations = [round_record.duration_seconds for round_record in result.rounds]
+    assert durations == pytest.approx([0.5e-6, 0.5e-6])
+    assert all(duration > 0 and math.isfinite(duration) for duration in durations)
+    assert sum(durations) == pytest.approx(1e-6)
+    assert result.raw_summary["proposal_duration_seconds"] == pytest.approx(1e-6)
+
+
 @pytest.mark.asyncio
 async def test_fake_backend_returns_no_candidates_without_failure_categories(tmp_path: Path):
     baseline_prompt = DEFAULT_PROMPT.read_text(encoding="utf-8")
@@ -1962,6 +2032,26 @@ def _empty_eval_result(prompt_id: str, split: str) -> EvalResult:
     )
 
 
+def _failed_fake_train_result() -> EvalResult:
+    return EvalResult(
+        prompt_id="baseline",
+        split="train",
+        score=0.0,
+        passed=False,
+        cost=0.0,
+        cases=[
+            CaseResult(
+                case_id="observed_training_failure",
+                split="train",
+                score=0.0,
+                passed=False,
+                output="not-json",
+                failure_category="format_violation",
+            )
+        ],
+    )
+
+
 def _sdk_round(
     round_id: int,
     candidate_prompts: dict[str, str],