ci: populate baselines and gate benchmarks at +20%

.github/workflows/ci.yml

@@ -207,7 +207,7 @@ jobs:
       - run: npm run test:coverage
 
   benchmarks:
-    name: Performance benchmarks (informational)
+    name: Performance benchmarks (gated)
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -235,6 +235,9 @@ jobs:
           --benchmark-columns=min,max,mean,stddev,rounds
           -o addopts=
 
+      - name: Regression gate
+        run: python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json
+
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: benchmark-results

.gitignore

@@ -14,3 +14,5 @@ node_modules/
 .coverage
 coverage/
 coverage.xml
+benchmark-results.json
+benchmarks/_raw.json

Makefile

@@ -0,0 +1,9 @@
+.PHONY: update-baselines check-benchmarks
+
+update-baselines:
+	pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
+	python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json
+
+check-benchmarks:
+	pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmark-results.json -o addopts=
+	python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json

benchmarks/README.md

@@ -1,6 +1,6 @@
 # Performance benchmarks
 
-Test files live under `tests/benchmarks/`; this directory holds only documentation and the informational `baselines.json` snapshot.
+Test files live under `tests/benchmarks/`; this directory holds documentation and `baselines.json` for the CI regression gate.
 
 Repeatable local measurements for parse, bulk export, and search hot paths.
 
@@ -26,17 +26,31 @@ The memory test also runs as part of the normal `pytest` suite (timing benchmark
 | parse | `parse_session` on 10 / 500 / 5000+ line JSONL |
 | export | `run_bulk_export` over 10 / 50 / 100 sessions |
 | search | `GET /api/search` over a 50-session synthetic corpus |
+| cache | cold vs warm `get_cached_session` (informational; not gated) |
 
 Large JSONL files (5000+ lines) are generated at test session scope under pytest's temp directory — not committed to git.
 
 Corpora repeat one row from `tests/fixtures/session_with_tools.jsonl`, so parse/export numbers measure steady-state throughput on a narrow schema slice — not full parser branch coverage. Treat as v1 baselines, not exhaustive perf proof.
 
 The memory test (`test_parse_memory.py`) is intentionally **not** skipped by `--benchmark-skip`; it runs in the main `pytest` job and builds the session-scoped 5000-line fixture once per session.
 
-## CI
+## CI gate
 
-The `benchmarks` workflow job uploads `benchmark-results.json` as a downloadable artifact. There is no regression gate yet.
+The `benchmarks` job on **ubuntu-latest** runs pytest-benchmark, then `scripts/check_benchmark_regression.py`. CI fails when any gated benchmark mean exceeds its baseline by more than **20%**. Benchmarks without a baseline entry (e.g. new `cache` group) print a warning and do not fail the gate.
 
 ## Refresh baselines
 
-After intentional performance work, copy key means from a local run into `baselines.json` with a date and machine note. This file is informational only; CI does not compare against it.
+After intentional performance work, capture on **ubuntu-latest** (same OS as the gated CI job):
+
+```bash
+make update-baselines
+```
+
+Or manually:
+
+```bash
+pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
+python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json
+```
+
+Do not capture baselines on Windows/macOS for commit — ubuntu runners are slower and the gate will fail. Download `benchmark-results.json` from a green CI artifact to seed baselines if needed.

benchmarks/baselines.json

@@ -1,10 +1,20 @@
 {
-  "_note": "Informational snapshot only — CI does not gate on these values.",
-  "updated": null,
-  "machine": null,
+  "_note": "Means captured from ubuntu-latest CI post-cache (PR #90). Refresh via make update-baselines on ubuntu.",
+  "updated": "2026-06-17T21:00:00Z",
+  "machine": "Linux",
   "groups": {
-    "parse": {},
-    "export": {},
-    "search": {}
+    "parse": {
+      "test_parse_session_small": 0.000105,
+      "test_parse_session_medium": 0.002956,
+      "test_parse_session_large": 0.029678
+    },
+    "export": {
+      "test_bulk_export_session_count[sessions-10]": 0.004278,
+      "test_bulk_export_session_count[sessions-50]": 0.021144,
+      "test_bulk_export_session_count[sessions-100]": 0.042003
+    },
+    "search": {
+      "test_search_full_corpus": 0.001092
+    }
   }
 }

scripts/check_benchmark_regression.py

@@ -0,0 +1,118 @@
+"""Compare pytest-benchmark JSON output against stored baselines."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+THRESHOLD = 1.20
+
+
+class BenchmarkDataError(ValueError):
+    """Raised when benchmark JSON input is malformed or missing required fields."""
+
+
+def load_results(results_path: str | Path) -> dict[str, float]:
+    path = Path(results_path)
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise BenchmarkDataError(f"invalid JSON in {path}: {exc}") from exc
+    try:
+        benchmarks = data["benchmarks"]
+    except (KeyError, TypeError) as exc:
+        raise BenchmarkDataError(f"{path} missing top-level 'benchmarks' array") from exc
+    if not isinstance(benchmarks, list):
+        raise BenchmarkDataError(f"{path} 'benchmarks' must be an array")
+
+    results: dict[str, float] = {}
+    for index, entry in enumerate(benchmarks):
+        if not isinstance(entry, dict):
+            raise BenchmarkDataError(f"{path} benchmarks[{index}] must be an object")
+        try:
+            name = entry["name"]
+            mean = entry["stats"]["mean"]
+        except (KeyError, TypeError) as exc:
+            raise BenchmarkDataError(
+                f"{path} benchmarks[{index}] missing 'name' or 'stats.mean'"
+            ) from exc
+        results[str(name)] = float(mean)
+    return results
+
+
+def load_baseline_means(baselines_path: str | Path) -> dict[str, float]:
+    path = Path(baselines_path)
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise BenchmarkDataError(f"invalid JSON in {path}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise BenchmarkDataError(f"{path} root value must be an object")
+
+    groups = data.get("groups", data)
+    if not isinstance(groups, dict):
+        raise BenchmarkDataError(f"{path} missing 'groups' object")
+
+    means: dict[str, float] = {}
+    for group_name, value in groups.items():
+        if not isinstance(value, dict):
+            continue
+        for name, mean in value.items():
+            try:
+                means[str(name)] = float(mean)
+            except (TypeError, ValueError) as exc:
+                raise BenchmarkDataError(
+                    f"{path} groups[{group_name!r}][{name!r}] is not a numeric mean"
+                ) from exc
+    return means
+
+
+def check_regression(
+    results_path: str | Path,
+    baselines_path: str | Path,
+    *,
+    threshold: float = THRESHOLD,
+) -> int:
+    """Return 0 when within threshold; 1 when any gated benchmark regresses."""
+    flat = load_results(results_path)
+    baseline_means = load_baseline_means(baselines_path)
+
+    failures: list[str] = []
+    for name, base in baseline_means.items():
+        cur = flat.get(name)
+        if cur is None:
+            print(f"WARN: no current result for baseline {name!r}; skipping")
+            continue
+        if base == 0:
+            print(f"WARN: baseline for {name!r} is zero; skipping ratio check")
+            continue
+        ratio = cur / base
+        tag = "FAIL" if ratio > threshold else "ok"
+        print(f"[{tag}] {name}: {cur:.6f}s vs {base:.6f}s ({ratio:.2f}x)")
+        if ratio > threshold:
+            failures.append(name)
+
+    for name in flat:
+        if name not in baseline_means:
+            print(f"WARN: {name!r} has no baseline yet; not gated")
+
+    if failures:
+        print(f"\nREGRESSION: {len(failures)} benchmark(s) exceeded {threshold:.0%}")
+        return 1
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    if len(argv) != 2:
+        print(
+            "usage: check_benchmark_regression.py <results.json> <baselines.json>",
+            file=sys.stderr,
+        )
+        return 2
+    return check_regression(argv[0], argv[1])
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/reduce_baselines.py

@@ -0,0 +1,63 @@
+"""Reduce pytest-benchmark JSON into benchmarks/baselines.json."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+GATED_GROUPS = ("parse", "export", "search")
+
+
+def _positive_float(value: str) -> float:
+    parsed = float(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("slack must be greater than zero")
+    return parsed
+
+
+def reduce_baselines(
+    raw_path: str | Path,
+    out_path: str | Path,
+    *,
+    slack: float = 1.0,
+) -> dict[str, object]:
+    raw = json.loads(Path(raw_path).read_text(encoding="utf-8"))
+    groups: dict[str, dict[str, float]] = {group: {} for group in GATED_GROUPS}
+    for entry in raw["benchmarks"]:
+        group = entry.get("group")
+        if group not in GATED_GROUPS:
+            continue
+        groups[group][entry["name"]] = float(entry["stats"]["mean"]) * slack
+
+    machine_info = raw.get("machine_info", {})
+    output: dict[str, object] = {
+        "_note": "CI gates the ubuntu benchmarks job when mean exceeds baseline by >20%.",
+        "updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "machine": machine_info.get("system"),
+        "groups": groups,
+    }
+    path = Path(out_path)
+    path.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
+    return output
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("raw_path", help="pytest-benchmark --benchmark-json output")
+    parser.add_argument("out_path", help="destination baselines.json path")
+    parser.add_argument(
+        "--slack",
+        type=_positive_float,
+        default=1.0,
+        help="multiply means by this factor (must be > 0)",
+    )
+    args = parser.parse_args(argv)
+    reduce_baselines(args.raw_path, args.out_path, slack=args.slack)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_check_benchmark_regression.py

@@ -0,0 +1,107 @@
+"""Tests for scripts/check_benchmark_regression.py."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from scripts.check_benchmark_regression import BenchmarkDataError, check_regression, load_results
+
+
+def _write_results(path, benchmarks: list[dict]) -> None:
+    path.write_text(
+        json.dumps({"benchmarks": benchmarks}, indent=2),
+        encoding="utf-8",
+    )
+
+
+def _write_baselines(path, groups: dict[str, dict[str, float]]) -> None:
+    path.write_text(
+        json.dumps({"groups": groups}, indent=2),
+        encoding="utf-8",
+    )
+
+
+def test_missing_baseline_warns_without_failing(
+    tmp_path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    results = tmp_path / "results.json"
+    baselines = tmp_path / "baselines.json"
+    _write_results(
+        results,
+        [
+            {"name": "test_new_bench", "stats": {"mean": 0.01}},
+            {"name": "test_parse_session_small", "stats": {"mean": 0.0001}},
+        ],
+    )
+    _write_baselines(
+        baselines,
+        {"parse": {"test_parse_session_small": 0.0001}},
+    )
+
+    assert check_regression(results, baselines) == 0
+    out = capsys.readouterr().out
+    assert "WARN: 'test_new_bench' has no baseline yet" in out
+
+
+def test_regression_over_threshold_fails(tmp_path, capsys: pytest.CaptureFixture[str]) -> None:
+    results = tmp_path / "results.json"
+    baselines = tmp_path / "baselines.json"
+    _write_results(
+        results,
+        [{"name": "test_parse_session_small", "stats": {"mean": 0.0002}}],
+    )
+    _write_baselines(
+        baselines,
+        {"parse": {"test_parse_session_small": 0.0001}},
+    )
+
+    assert check_regression(results, baselines) == 1
+    out = capsys.readouterr().out
+    assert "REGRESSION" in out
+
+
+def test_within_threshold_passes(tmp_path) -> None:
+    results = tmp_path / "results.json"
+    baselines = tmp_path / "baselines.json"
+    _write_results(
+        results,
+        [{"name": "test_parse_session_small", "stats": {"mean": 0.00011}}],
+    )
+    _write_baselines(
+        baselines,
+        {"parse": {"test_parse_session_small": 0.0001}},
+    )
+
+    assert check_regression(results, baselines) == 0
+
+
+def test_load_results_rejects_malformed_json(tmp_path) -> None:
+    path = tmp_path / "bad.json"
+    path.write_text("{not json", encoding="utf-8")
+    with pytest.raises(BenchmarkDataError, match="invalid JSON"):
+        load_results(path)
+
+
+def test_load_results_requires_benchmarks_array(tmp_path) -> None:
+    path = tmp_path / "results.json"
+    path.write_text("{}", encoding="utf-8")
+    with pytest.raises(BenchmarkDataError, match="'benchmarks' array"):
+        load_results(path)
+
+
+def test_zero_baseline_skips_ratio_check(tmp_path, capsys: pytest.CaptureFixture[str]) -> None:
+    results = tmp_path / "results.json"
+    baselines = tmp_path / "baselines.json"
+    _write_results(
+        results,
+        [{"name": "test_parse_session_small", "stats": {"mean": 0.0002}}],
+    )
+    _write_baselines(
+        baselines,
+        {"parse": {"test_parse_session_small": 0.0}},
+    )
+
+    assert check_regression(results, baselines) == 0
+    assert "baseline for 'test_parse_session_small' is zero" in capsys.readouterr().out