fix(test): harden benchmark gate and tracemalloc fixtures (PR review)

Author: clean6378-max-it

Makefile

@@ -1,12 +1,19 @@
-.PHONY: update-baselines check-benchmarks clean-benchmark-artifacts
+.PHONY: seed-baselines-local update-baselines check-benchmarks clean-benchmark-artifacts
 
-update-baselines:
-	pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
-	python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json
+# WARNING: captures timings on THIS machine. Production baselines must match ubuntu-latest CI.
+# Prefer downloading benchmark-results.json from a CI artifact, then:
+#   python scripts/reduce_baselines.py benchmark-results.json benchmarks/baselines.json --slack 1.5
+seed-baselines-local:
+	@echo "WARNING: seed-baselines-local uses this host's timings; CI gates on ubuntu-latest." >&2
+	PYTHONPATH=. pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
+	PYTHONPATH=. python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json
+
+# Deprecated alias — kept for muscle memory; see seed-baselines-local warning above.
+update-baselines: seed-baselines-local
 
 check-benchmarks:
-	pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmark-results.json -o addopts=
-	python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json
+	PYTHONPATH=. pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmark-results.json -o addopts=
+	PYTHONPATH=. python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json
 
 clean-benchmark-artifacts:
 	rm -f benchmarks/_raw.json benchmark-results.json

benchmarks/README.md

@@ -40,21 +40,31 @@ The `benchmarks` job on **ubuntu-latest** runs pytest-benchmark (`--benchmark-js
 
 **Gated:** parse medium/large + large peak memory; export 10/50/100 session latency + ZIP peak memory.
 
-**Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. Benchmarks without a baseline entry print a warning and do not fail the gate.
+**Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. These names are omitted from `baselines.json` when using `reduce_baselines.py`. Benchmarks without a baseline entry print a warning and do not fail the gate.
+
+Missing gated benchmarks (renamed or removed tests still listed in `baselines.json`) fail the gate.
 
 ## Refresh baselines
 
-After intentional performance work, capture on **ubuntu-latest** (same OS as the gated CI job):
+After intentional performance work, capture on **ubuntu-latest** (same OS as the gated CI job). Download `benchmark-results.json` from a CI artifact when possible:
+
+```bash
+python scripts/reduce_baselines.py benchmark-results.json benchmarks/baselines.json --slack 1.5
+```
+
+For a quick local snapshot only (may not match CI timings):
 
 ```bash
-make update-baselines
+make seed-baselines-local
 ```
 
+`make update-baselines` is a deprecated alias for `seed-baselines-local` and prints a warning. Do not commit baselines from macOS/Windows unless you accept cross-OS gate skew.
+
 Or manually:
 
 ```bash
-pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
-python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json
+PYTHONPATH=. pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
+PYTHONPATH=. python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json
 ```
 
-Baselines must be captured on **ubuntu-latest** to match the gated CI runner. Cross-OS variance causes spurious failures. Download `benchmark-results.json` from a CI artifact to seed baselines if needed.
+Baselines must be captured on **ubuntu-latest** to match the gated CI runner. Cross-OS variance causes spurious failures.

benchmarks/baselines.json

@@ -1,5 +1,5 @@
 {
-  "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR memory-path benchmarks). Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).",
+  "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR memory-path benchmarks). Values multiplied by 1.5× slack at seeding time. Excluded from gate (not in this file): test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).",
   "updated": "2026-06-25T12:00:00Z",
   "machine": "Linux",
   "groups": {
@@ -15,9 +15,6 @@
       "test_bulk_export_zip_peak_memory[sessions-10]": 350546.0,
       "test_bulk_export_zip_peak_memory[sessions-50]": 503331.0,
       "test_bulk_export_zip_peak_memory[sessions-100]": 686874.0
-    },
-    "search": {
-      "test_search_full_corpus": 0.001091
     }
   }
 }

scripts/check_benchmark_regression.py

@@ -39,8 +39,16 @@ def benchmark_entry_mean(entry: dict[str, object]) -> float:
     """Return gated metric: peak_bytes from extra_info when present, else stats.mean."""
     if entry_uses_peak_bytes(entry):
         extra = entry["extra_info"]
-        assert isinstance(extra, dict)
-        return float(extra["peak_bytes"])
+        if not isinstance(extra, dict):
+            raise BenchmarkDataError(
+                f"extra_info for {entry.get('name')!r} is not a dict"
+            )
+        try:
+            return float(extra["peak_bytes"])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise BenchmarkDataError(
+                f"benchmark {entry.get('name')!r} missing 'stats.mean' or extra_info.peak_bytes"
+            ) from exc
     try:
         stats = entry["stats"]
         return float(stats["mean"])  # type: ignore[index]
@@ -75,6 +83,8 @@ def load_results(
         try:
             name = entry["name"]
             mean = benchmark_entry_mean(entry)
+        except BenchmarkDataError:
+            raise
         except (KeyError, TypeError, ValueError) as exc:
             raise BenchmarkDataError(
                 f"{path} benchmarks[{index}] missing 'name' or measurable value"
@@ -132,12 +142,14 @@ def check_regression(
     baseline_means = load_baseline_means(baselines_path)
 
     failures: list[str] = []
+    missing: list[str] = []
     for name, base in baseline_means.items():
         if name in EXCLUDED_FROM_GATE:
             continue
         cur = flat.get(name)
         if cur is None:
-            print(f"WARN: no current result for baseline {name!r}; skipping")
+            print(f"FAIL: no current result for gated baseline {name!r}")
+            missing.append(name)
             continue
         if base == 0:
             print(f"WARN: baseline for {name!r} is zero; skipping ratio check")
@@ -160,6 +172,9 @@ def check_regression(
 
     if failures:
         print(f"\nREGRESSION: {len(failures)} benchmark(s) exceeded {threshold:.0%}")
+    if missing:
+        print(f"\nMISSING: {len(missing)} gated benchmark(s) absent from current results")
+    if failures or missing:
         return 1
     return 0
 

scripts/reduce_baselines.py

@@ -8,10 +8,11 @@
 from datetime import UTC, datetime
 from pathlib import Path
 
-try:
-    from scripts.check_benchmark_regression import BenchmarkDataError, benchmark_entry_mean
-except ModuleNotFoundError:
-    from check_benchmark_regression import BenchmarkDataError, benchmark_entry_mean
+from scripts.check_benchmark_regression import (
+    EXCLUDED_FROM_GATE,
+    BenchmarkDataError,
+    benchmark_entry_mean,
+)
 
 GATED_GROUPS = ("parse", "export", "search")
 
@@ -51,21 +52,32 @@ def reduce_baselines(
         try:
             name = entry["name"]
             mean = benchmark_entry_mean(entry)
+        except BenchmarkDataError:
+            raise
         except (KeyError, TypeError, ValueError) as exc:
             raise BenchmarkDataError(
                 f"{path} benchmarks[{index}] missing 'name' or measurable value"
             ) from exc
+        bench_name = str(name)
+        if bench_name in EXCLUDED_FROM_GATE:
+            continue
         group = entry.get("group")
         if group not in GATED_GROUPS:
             continue
-        groups[group][str(name)] = mean * slack
+        groups[group][bench_name] = mean * slack
+
+    # Drop empty groups (e.g. search when only excluded benchmarks ran).
+    groups = {name: values for name, values in groups.items() if values}
 
+    slack_note = f" Values multiplied by {slack}× slack at generation time." if slack != 1.0 else ""
     machine_info = raw.get("machine_info")
     machine = machine_info.get("system") if isinstance(machine_info, dict) else None
     output: dict[str, object] = {
         "_note": (
-            "Gated means from ubuntu-latest CI benchmark-results.json. "
-            "Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). "
+            "Gated means from ubuntu-latest CI benchmark-results.json."
+            f"{slack_note} "
+            "Excluded from gate (not written here): test_parse_session_small, "
+            "test_search_full_corpus (CI noise). "
             "Memory benchmarks use extra_info.peak_bytes (bytes); "
             "latency uses stats.mean (seconds)."
         ),

tests/benchmarks/conftest.py

@@ -23,28 +23,33 @@ class TracemallocPeak:
     """Measure peak Python heap bytes for one callable invocation."""
 
     def measure(self, func: Callable[..., T], /, *args: Any, **kwargs: Any) -> tuple[T, int]:
+        was_tracing = tracemalloc.is_tracing()
         tracemalloc.start()
         tracemalloc.clear_traces()
         try:
             result = func(*args, **kwargs)
             _, peak = tracemalloc.get_traced_memory()
             return result, peak
         finally:
-            tracemalloc.stop()
+            if not was_tracing:
+                tracemalloc.stop()
 
 
 @pytest.fixture
 def tracemalloc_peak() -> TracemallocPeak:
     return TracemallocPeak()
 
 
-def write_jsonl(path: Path, line_count: int) -> Path:
+def write_jsonl(path: Path, line_count: int, *, first_timestamp: str | None = None) -> Path:
     """Write a JSONL session file with *line_count* rows derived from the template fixture."""
     template = json.loads(TEMPLATE_LINE)
     with path.open("w", encoding="utf-8") as f:
         for i in range(line_count):
             entry = deepcopy(template)
-            entry["timestamp"] = f"2026-06-12T10:{i % 60:02d}:00Z"
+            if i == 0 and first_timestamp is not None:
+                entry["timestamp"] = first_timestamp
+            else:
+                entry["timestamp"] = f"2026-06-12T10:{i % 60:02d}:00Z"
             if i % 3 == 1:
                 msg = entry.setdefault("message", {})
                 if isinstance(msg, dict) and "content" in msg:
@@ -96,7 +101,9 @@ def export_corpus(tmp_path: Path, request: pytest.FixtureRequest) -> Path:
     project = tmp_path / "bench-project"
     project.mkdir()
     for i in range(count):
-        write_jsonl(project / f"session_{i:04d}.jsonl", 20)
+        # Unique first_timestamp per session so export filenames do not collide in ZIP benches.
+        first_ts = f"2026-06-12T{i % 24:02d}:{i % 60:02d}:00Z"
+        write_jsonl(project / f"session_{i:04d}.jsonl", 20, first_timestamp=first_ts)
     return project
 
 

tests/benchmarks/test_export_bench.py

@@ -8,7 +8,6 @@
 
 import pytest
 
-from tests.benchmarks.conftest import TracemallocPeak
 from utils.export_engine import BulkExportResult, NoopSink, ZipSink, run_bulk_export
 
 
@@ -56,10 +55,11 @@ def _run() -> object:
 def test_bulk_export_zip_peak_memory(
     benchmark,
     export_corpus: Path,
-    tracemalloc_peak: TracemallocPeak,
+    tracemalloc_peak,
 ) -> None:
     projects = _bench_projects(export_corpus)
     peaks: list[int] = []
+    results: list[BulkExportResult] = []
 
     def _run() -> None:
         def _export() -> BulkExportResult:
@@ -78,9 +78,10 @@ def _export() -> BulkExportResult:
                 )
 
         result, peak = tracemalloc_peak.measure(_export)
-        assert result.exported_session_count > 0
+        results.append(result)
         peaks.append(peak)
 
     benchmark(_run)
+    assert results and results[-1].exported_session_count > 0
     assert peaks, "benchmark produced no peak memory samples"
     benchmark.extra_info["peak_bytes"] = int(sum(peaks) / len(peaks))

tests/benchmarks/test_parse_memory.py

@@ -2,40 +2,34 @@
 
 from __future__ import annotations
 
-import tracemalloc
 from pathlib import Path
 
 import pytest
 
-from tests.benchmarks.conftest import TracemallocPeak
 from utils.jsonl_parser import parse_session
 
 
-def test_large_parse_peak_memory_under_ceiling(parse_large_file: Path) -> None:
+def test_large_parse_peak_memory_under_ceiling(
+    parse_large_file: Path,
+    tracemalloc_peak,
+) -> None:
     path = parse_large_file
     file_bytes = path.stat().st_size
     # Issue #7 ceiling: Python heap peak (tracemalloc) vs on-disk JSONL size. Parsed
     # dict/str objects often exceed raw bytes; 10x is a generous v1 guard — relax with
     # a comment here if the parser legitimately grows.
     ceiling = file_bytes * 10
 
-    tracemalloc.start()
-    tracemalloc.clear_traces()
-    try:
-        result = parse_session(str(path))
-        assert len(result["messages"]) > 0, "parse_session returned no messages"
-        _, peak = tracemalloc.get_traced_memory()
-    finally:
-        tracemalloc.stop()
-
+    result, peak = tracemalloc_peak.measure(parse_session, str(path))
+    assert len(result["messages"]) > 0, "parse_session returned no messages"
     assert peak < ceiling, f"peak {peak} bytes exceeds 10x file size {file_bytes}"
 
 
 @pytest.mark.benchmark(group="parse")
 def test_parse_large_peak_memory(
     benchmark,
     parse_large_file: Path,
-    tracemalloc_peak: TracemallocPeak,
+    tracemalloc_peak,
 ) -> None:
     path = str(parse_large_file)
     peaks: list[int] = []

tests/test_check_benchmark_regression.py

@@ -39,12 +39,12 @@ def test_missing_baseline_warns_without_failing(
         results,
         [
             {"name": "test_new_bench", "stats": {"mean": 0.01}},
-            {"name": "test_parse_session_small", "stats": {"mean": 0.0001}},
+            {"name": GATED_BENCH, "stats": {"mean": 0.002}},
         ],
     )
     _write_baselines(
         baselines,
-        {"parse": {"test_parse_session_small": 0.0001}},
+        {"parse": {GATED_BENCH: 0.002}},
     )
 
     assert check_regression(results, baselines) == 0
@@ -152,9 +152,7 @@ def test_excluded_benchmark_in_baselines_is_not_gated(
     assert "REGRESSION" not in capsys.readouterr().out
 
 
-def test_missing_current_result_warns_without_failing(
-    tmp_path, capsys: pytest.CaptureFixture[str]
-) -> None:
+def test_missing_current_result_fails(tmp_path, capsys: pytest.CaptureFixture[str]) -> None:
     results = tmp_path / "results.json"
     baselines = tmp_path / "baselines.json"
     _write_results(results, [])
@@ -163,8 +161,10 @@ def test_missing_current_result_warns_without_failing(
         {"parse": {GATED_BENCH: 0.002}},
     )
 
-    assert check_regression(results, baselines) == 0
-    assert "no current result for baseline" in capsys.readouterr().out
+    assert check_regression(results, baselines) == 1
+    out = capsys.readouterr().out
+    assert "MISSING" in out
+    assert "no current result for gated baseline" in out
 
 
 def test_main_reports_benchmark_data_error(tmp_path, capsys: pytest.CaptureFixture[str]) -> None:
@@ -199,11 +199,11 @@ def test_metric_is_bytes_uses_extra_info_without_name_hint() -> None:
     from scripts.check_benchmark_regression import metric_is_bytes
 
     entry = {
-        "name": "test_custom_memory",
+        "name": "test_export_latency",
         "stats": {"mean": 0.05},
         "extra_info": {"peak_bytes": 1_000_000},
     }
-    assert metric_is_bytes("test_custom_memory", entry)
+    assert metric_is_bytes("test_export_latency", entry)
 
 
 def test_memory_metric_regression_uses_bytes(tmp_path, capsys: pytest.CaptureFixture[str]) -> None:
@@ -230,6 +230,29 @@ def test_memory_metric_regression_uses_bytes(tmp_path, capsys: pytest.CaptureFix
     assert "REGRESSION" in out
 
 
+def test_benchmark_entry_mean_rejects_non_dict_extra_info() -> None:
+    from scripts.check_benchmark_regression import benchmark_entry_mean
+
+    with pytest.raises(BenchmarkDataError, match="extra_info"):
+        benchmark_entry_mean(
+            {
+                "name": "test_parse_large_peak_memory",
+                "extra_info": "not-a-dict",
+            }
+        )
+
+
+def test_load_results_preserves_benchmark_data_error_message(tmp_path) -> None:
+    path = tmp_path / "results.json"
+    _write_results(
+        path,
+        [{"name": "test_parse_large_peak_memory", "extra_info": {"peak_bytes": "bad"}}],
+    )
+
+    with pytest.raises(BenchmarkDataError, match="extra_info.peak_bytes"):
+        load_results(path)
+
+
 def test_duplicate_baseline_name_raises(tmp_path) -> None:
     baselines = tmp_path / "baselines.json"
     _write_baselines(