test: gate memory-path benchmarks in baselines (parse + ZIP export)

Author: clean6378-max-it

benchmarks/README.md

@@ -23,22 +23,22 @@ The memory test also runs as part of the normal `pytest` suite (timing benchmark
 
 | Group | What |
 |-------|------|
-| parse | `parse_session` on 10 / 500 / 5000+ line JSONL |
-| export | `run_bulk_export` over 10 / 50 / 100 sessions |
+| parse | `parse_session` on 10 / 500 / 5000+ line JSONL; large-file peak heap (`test_parse_large_peak_memory`) |
+| export | `run_bulk_export` latency over 10 / 50 / 100 sessions; ZIP export peak heap (`test_bulk_export_zip_peak_memory`) |
 | search | `GET /api/search` over a 50-session synthetic corpus |
 | cache | cold vs warm `get_cached_session` (informational; not gated) |
 
 Large JSONL files (5000+ lines) are generated at test session scope under pytest's temp directory — not committed to git.
 
 Corpora repeat one row from `tests/fixtures/session_with_tools.jsonl`, so parse/export numbers measure steady-state throughput on a narrow schema slice — not full parser branch coverage. Treat as v1 baselines, not exhaustive perf proof.
 
-The memory test (`test_parse_memory.py`) is intentionally **not** skipped by `--benchmark-skip`; it runs in the main `pytest` job and builds the session-scoped 5000-line fixture once per session.
+The memory ceiling test (`test_large_parse_peak_memory_under_ceiling`) runs in the main `pytest` job. Tracked peak-memory benchmarks (`test_parse_large_peak_memory`, `test_bulk_export_zip_peak_memory`) run under `--benchmark-only` and store `extra_info.peak_bytes` for the regression gate.
 
 ## CI gate
 
 The `benchmarks` job on **ubuntu-latest** runs pytest-benchmark (`--benchmark-json=benchmark-results.json`), then `scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json`. CI fails when any **gated** benchmark mean exceeds its baseline by more than **20%**.
 
-**Gated:** parse medium/large, export 10/50/100 sessions.
+**Gated:** parse medium/large + large peak memory; export 10/50/100 session latency + ZIP peak memory.
 
 **Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. Benchmarks without a baseline entry print a warning and do not fail the gate.
 

benchmarks/baselines.json

@@ -1,17 +1,23 @@
 {
-  "_note": "Gated means from ubuntu-latest CI benchmark-results.json (post-cache PR #90). Excluded from gate: test_parse_session_small, test_search_full_corpus (sub-ms CI noise). Refresh via make update-baselines on ubuntu.",
-  "updated": "2026-06-17T21:00:00Z",
-  "machine": "Linux",
+  "_note": "Gated means from local reference run with 1.5x slack (Windows dev host). Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use peak_bytes; latency uses stats.mean. Refresh from ubuntu-latest CI artifact after first green job.",
+  "updated": "2026-06-25T00:00:00Z",
+  "machine": "Windows",
   "groups": {
     "parse": {
-      "test_parse_session_medium": 0.002956,
-      "test_parse_session_large": 0.029678
+      "test_parse_session_medium": 0.0021930547314696017,
+      "test_parse_session_large": 0.022708179059622507,
+      "test_parse_large_peak_memory": 3175761.0
     },
     "export": {
-      "test_bulk_export_session_count[sessions-10]": 0.004278,
-      "test_bulk_export_session_count[sessions-50]": 0.021144,
-      "test_bulk_export_session_count[sessions-100]": 0.042003
+      "test_bulk_export_session_count[sessions-10]": 0.003750986296823886,
+      "test_bulk_export_session_count[sessions-50]": 0.018420866638835933,
+      "test_bulk_export_session_count[sessions-100]": 0.03871899230244498,
+      "test_bulk_export_zip_peak_memory[sessions-10]": 525699.0,
+      "test_bulk_export_zip_peak_memory[sessions-50]": 752907.0,
+      "test_bulk_export_zip_peak_memory[sessions-100]": 1029018.0
     },
-    "search": {}
+    "search": {
+      "test_search_full_corpus": 0.0033342776477665584
+    }
   }
 }

scripts/check_benchmark_regression.py

@@ -22,6 +22,24 @@ class BenchmarkDataError(ValueError):
     """Raised when benchmark JSON input is malformed or missing required fields."""
 
 
+def benchmark_entry_mean(entry: dict[str, object]) -> float:
+    """Return gated metric: peak_bytes from extra_info when present, else stats.mean."""
+    extra = entry.get("extra_info")
+    if isinstance(extra, dict) and "peak_bytes" in extra:
+        return float(extra["peak_bytes"])
+    try:
+        stats = entry["stats"]
+        return float(stats["mean"])  # type: ignore[index]
+    except (KeyError, TypeError, ValueError) as exc:
+        raise BenchmarkDataError(
+            f"benchmark {entry.get('name')!r} missing 'stats.mean' or extra_info.peak_bytes"
+        ) from exc
+
+
+def is_memory_metric(name: str) -> bool:
+    return "peak_memory" in name
+
+
 def load_results(results_path: str | Path) -> dict[str, float]:
     path = Path(results_path)
     try:
@@ -43,10 +61,10 @@ def load_results(results_path: str | Path) -> dict[str, float]:
             raise BenchmarkDataError(f"{path} benchmarks[{index}] must be an object")
         try:
             name = entry["name"]
-            mean = float(entry["stats"]["mean"])
+            mean = benchmark_entry_mean(entry)
         except (KeyError, TypeError, ValueError) as exc:
             raise BenchmarkDataError(
-                f"{path} benchmarks[{index}] missing 'name' or 'stats.mean'"
+                f"{path} benchmarks[{index}] missing 'name' or measurable value"
             ) from exc
         name = str(name)
         if name in results:
@@ -112,7 +130,10 @@ def check_regression(
             continue
         ratio = cur / base
         tag = "FAIL" if ratio > threshold else "ok"
-        print(f"[{tag}] {name}: {cur:.6f}s vs {base:.6f}s ({ratio:.2f}x)")
+        if is_memory_metric(name):
+            print(f"[{tag}] {name}: {cur:.0f} bytes vs {base:.0f} bytes ({ratio:.2f}x)")
+        else:
+            print(f"[{tag}] {name}: {cur:.6f}s vs {base:.6f}s ({ratio:.2f}x)")
         if ratio > threshold:
             failures.append(name)
 

scripts/reduce_baselines.py

@@ -9,9 +9,17 @@
 from pathlib import Path
 
 try:
-    from scripts.check_benchmark_regression import EXCLUDED_FROM_GATE, BenchmarkDataError
+    from scripts.check_benchmark_regression import (
+        EXCLUDED_FROM_GATE,
+        BenchmarkDataError,
+        benchmark_entry_mean,
+    )
 except ModuleNotFoundError:
-    from check_benchmark_regression import EXCLUDED_FROM_GATE, BenchmarkDataError
+    from check_benchmark_regression import (
+        EXCLUDED_FROM_GATE,
+        BenchmarkDataError,
+        benchmark_entry_mean,
+    )
 
 GATED_GROUPS = ("parse", "export", "search")
 
@@ -50,24 +58,23 @@ def reduce_baselines(
             raise BenchmarkDataError(f"{path} benchmarks[{index}] must be an object")
         try:
             name = entry["name"]
-            mean = float(entry["stats"]["mean"])
+            mean = benchmark_entry_mean(entry)
         except (KeyError, TypeError, ValueError) as exc:
             raise BenchmarkDataError(
-                f"{path} benchmarks[{index}] missing 'name' or 'stats.mean'"
+                f"{path} benchmarks[{index}] missing 'name' or measurable value"
             ) from exc
         group = entry.get("group")
         if group not in GATED_GROUPS:
             continue
-        if str(name) in EXCLUDED_FROM_GATE:
-            continue
         groups[group][str(name)] = mean * slack
 
     machine_info = raw.get("machine_info")
     machine = machine_info.get("system") if isinstance(machine_info, dict) else None
     output: dict[str, object] = {
         "_note": (
             "Gated means from ubuntu-latest CI (post-cache). "
-            "Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise)."
+            "Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). "
+            "Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds)."
         ),
         "updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
         "machine": machine,

tests/benchmarks/conftest.py

@@ -3,8 +3,11 @@
 from __future__ import annotations
 
 import json
+import tracemalloc
+from collections.abc import Callable
 from copy import deepcopy
 from pathlib import Path
+from typing import Any, TypeVar
 
 import pytest
 
@@ -13,6 +16,27 @@
 FIXTURES = Path(__file__).resolve().parents[1] / "fixtures"
 TEMPLATE_LINE = (FIXTURES / "session_with_tools.jsonl").read_text(encoding="utf-8").splitlines()[0]
 
+T = TypeVar("T")
+
+
+class TracemallocPeak:
+    """Measure peak Python heap bytes for one callable invocation."""
+
+    def measure(self, func: Callable[..., T], /, *args: Any, **kwargs: Any) -> tuple[T, int]:
+        tracemalloc.start()
+        tracemalloc.clear_traces()
+        try:
+            result = func(*args, **kwargs)
+            _, peak = tracemalloc.get_traced_memory()
+            return result, peak
+        finally:
+            tracemalloc.stop()
+
+
+@pytest.fixture
+def tracemalloc_peak() -> TracemallocPeak:
+    return TracemallocPeak()
+
 
 def write_jsonl(path: Path, line_count: int) -> Path:
     """Write a JSONL session file with *line_count* rows derived from the template fixture."""

tests/benchmarks/test_export_bench.py

@@ -2,11 +2,18 @@
 
 from __future__ import annotations
 
+import io
+import zipfile
 from pathlib import Path
 
 import pytest
 
-from utils.export_engine import NoopSink, run_bulk_export
+from tests.benchmarks.conftest import TracemallocPeak
+from utils.export_engine import NoopSink, ZipSink, run_bulk_export
+
+
+def _bench_projects(export_corpus: Path) -> list[dict[str, str]]:
+    return [{"name": "bench-project", "path": str(export_corpus), "display_name": "Bench"}]
 
 
 @pytest.mark.benchmark(group="export")
@@ -20,7 +27,7 @@ def test_bulk_export_session_count(
     benchmark,
     export_corpus: Path,
 ) -> None:
-    projects = [{"name": "bench-project", "path": str(export_corpus), "display_name": "Bench"}]
+    projects = _bench_projects(export_corpus)
 
     def _run() -> object:
         # NoopSink + since="all" + empty last_export_sessions: no disk/state writes per round.
@@ -37,3 +44,43 @@ def _run() -> object:
 
     result = benchmark(_run)
     assert result.exported_session_count > 0
+
+
+@pytest.mark.benchmark(group="export")
+@pytest.mark.parametrize(
+    "export_corpus",
+    [10, 50, 100],
+    indirect=True,
+    ids=["sessions-10", "sessions-50", "sessions-100"],
+)
+def test_bulk_export_zip_peak_memory(
+    benchmark,
+    export_corpus: Path,
+    tracemalloc_peak: TracemallocPeak,
+) -> None:
+    projects = _bench_projects(export_corpus)
+    peaks: list[int] = []
+
+    def _run() -> None:
+        def _export() -> object:
+            buf = io.BytesIO()
+            with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+                sink = ZipSink(zf)
+                return run_bulk_export(
+                    projects=projects,
+                    since="all",
+                    rules=[],
+                    last_export_sessions={},
+                    sink=sink,
+                    fmt="md",
+                    path_layout="api",
+                    manifest_style="api",
+                )
+
+        result, peak = tracemalloc_peak.measure(_export)
+        assert result.exported_session_count > 0
+        peaks.append(peak)
+
+    benchmark(_run)
+    assert peaks, "benchmark produced no peak memory samples"
+    benchmark.extra_info["peak_bytes"] = int(sum(peaks) / len(peaks))

tests/benchmarks/test_parse_memory.py

@@ -1,10 +1,13 @@
-"""Peak memory ceiling for large-file parse_session (regular pytest, not benchmark-only)."""
+"""Peak memory for large-file parse_session: ceiling test + tracked benchmark."""
 
 from __future__ import annotations
 
 import tracemalloc
 from pathlib import Path
 
+import pytest
+
+from tests.benchmarks.conftest import TracemallocPeak
 from utils.jsonl_parser import parse_session
 
 
@@ -26,3 +29,21 @@ def test_large_parse_peak_memory_under_ceiling(parse_large_file: Path) -> None:
         tracemalloc.stop()
 
     assert peak < ceiling, f"peak {peak} bytes exceeds 10x file size {file_bytes}"
+
+
+@pytest.mark.benchmark(group="parse")
+def test_parse_large_peak_memory(
+    benchmark,
+    parse_large_file: Path,
+    tracemalloc_peak: TracemallocPeak,
+) -> None:
+    path = str(parse_large_file)
+    peaks: list[int] = []
+
+    def _run() -> None:
+        _, peak = tracemalloc_peak.measure(parse_session, path)
+        peaks.append(peak)
+
+    benchmark(_run)
+    assert peaks, "benchmark produced no peak memory samples"
+    benchmark.extra_info["peak_bytes"] = int(sum(peaks) / len(peaks))

tests/test_check_benchmark_regression.py

@@ -179,6 +179,46 @@ def test_main_reports_benchmark_data_error(tmp_path, capsys: pytest.CaptureFixtu
     assert "ERROR:" in capsys.readouterr().err
 
 
+def test_load_results_prefers_peak_bytes_extra_info(tmp_path) -> None:
+    path = tmp_path / "results.json"
+    _write_results(
+        path,
+        [
+            {
+                "name": "test_parse_large_peak_memory",
+                "stats": {"mean": 0.05},
+                "extra_info": {"peak_bytes": 12_345_678},
+            }
+        ],
+    )
+
+    assert load_results(path)["test_parse_large_peak_memory"] == 12_345_678.0
+
+
+def test_memory_metric_regression_uses_bytes(tmp_path, capsys: pytest.CaptureFixture[str]) -> None:
+    results = tmp_path / "results.json"
+    baselines = tmp_path / "baselines.json"
+    _write_results(
+        results,
+        [
+            {
+                "name": "test_parse_large_peak_memory",
+                "stats": {"mean": 0.05},
+                "extra_info": {"peak_bytes": 15_000_000},
+            }
+        ],
+    )
+    _write_baselines(
+        baselines,
+        {"parse": {"test_parse_large_peak_memory": 10_000_000}},
+    )
+
+    assert check_regression(results, baselines) == 1
+    out = capsys.readouterr().out
+    assert "bytes" in out
+    assert "REGRESSION" in out
+
+
 def test_duplicate_baseline_name_raises(tmp_path) -> None:
     baselines = tmp_path / "baselines.json"
     _write_baselines(

tests/test_reduce_baselines.py

@@ -39,7 +39,7 @@ def test_reduce_baselines_writes_gated_groups_only(tmp_path) -> None:
 
     assert output["machine"] == "Linux"
     assert "test_parse_session_medium" in output["groups"]["parse"]
-    assert "test_parse_session_small" not in output["groups"]["parse"]
+    assert "test_parse_session_small" in output["groups"]["parse"]
     assert "cache" not in output["groups"]
 
 
@@ -79,6 +79,26 @@ def test_reduce_baselines_cli_rejects_non_positive_slack(tmp_path) -> None:
     assert exc_info.value.code == 2
 
 
+def test_reduce_baselines_uses_peak_bytes_extra_info(tmp_path) -> None:
+    raw = tmp_path / "raw.json"
+    out = tmp_path / "baselines.json"
+    _write_raw(
+        raw,
+        [
+            {
+                "group": "parse",
+                "name": "test_parse_large_peak_memory",
+                "stats": {"mean": 0.05},
+                "extra_info": {"peak_bytes": 10_000_000},
+            }
+        ],
+    )
+
+    output = reduce_baselines(raw, out)
+
+    assert output["groups"]["parse"]["test_parse_large_peak_memory"] == 10_000_000.0
+
+
 def test_reduce_baselines_machine_info_non_dict(tmp_path) -> None:
     raw = tmp_path / "raw.json"
     raw.write_text(