ci: unified benchmark suite with full baselines and regression gate

.github/workflows/tests.yml

@@ -215,7 +215,7 @@ jobs:
             --redact \
             --exit-code 1
 
-  # ── Performance benchmarks: summary cache (issue #115) ─────────────────────
+  # ── Performance benchmarks: unified suite (issues #115, #110) ──────────────
   benchmarks:
     name: Performance benchmarks (gated)
     needs: [unittest]
@@ -236,7 +236,7 @@ jobs:
           python -m pip install -r requirements-lock.txt
           python -m pip install 'pytest>=8,<9' 'pytest-benchmark==4.0.0'
 
-      - name: Run summary-cache benchmarks
+      - name: Run benchmark suite
         run: >
           python -m pytest tests/benchmarks/
           --benchmark-only

.gitignore

@@ -46,3 +46,4 @@ coverage.xml
 .hypothesis/
 benchmark-results.json
 benchmarks/_raw.json
+benchmarks/_ci/

Makefile

@@ -0,0 +1,19 @@
+.PHONY: seed-baselines-local update-baselines check-benchmarks clean-benchmark-artifacts
+
+# WARNING: captures timings on THIS machine. Production baselines must match ubuntu-latest CI.
+# Prefer downloading benchmark-results.json from a CI artifact, then:
+#   python scripts/reduce_baselines.py benchmark-results.json benchmarks/baselines.json --slack 1.5
+seed-baselines-local:
+	@echo "WARNING: seed-baselines-local uses this host's timings; CI gates on ubuntu-latest." >&2
+	python -m pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
+	python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json --slack 1.5
+
+# Deprecated alias — kept for muscle memory; see seed-baselines-local warning above.
+update-baselines: seed-baselines-local
+
+check-benchmarks:
+	python -m pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmark-results.json -o addopts=
+	python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json
+
+clean-benchmark-artifacts:
+	rm -f benchmarks/_raw.json benchmark-results.json

benchmarks/README.md

@@ -0,0 +1,60 @@
+# Performance benchmarks
+
+Test files live under `tests/benchmarks/`; this directory holds documentation and `baselines.json` for the CI regression gate.
+
+Repeatable local measurements for workspace listing, export, search, and summary-cache hot paths.
+
+## Run locally
+
+```bash
+pip install -r requirements-lock.txt
+pip install 'pytest>=8,<9' 'pytest-benchmark==4.0.0'
+pytest tests/benchmarks/ --benchmark-only -o addopts= -v
+```
+
+## Scenarios
+
+| Group | What |
+|-------|------|
+| parse | `list_workspace_projects(..., nocache=True)` over 10 / 50 / 200 synthetic composers |
+| export | `POST /api/export` (ZIP) over 10 / 50 composer corpora |
+| search | `GET /api/search` over a 50-composer synthetic corpus |
+| summary-cache | cache lookup (hit/miss), fingerprint (10/50/200), round-trip, tab-summary lookup |
+
+Synthetic corpora are built in `tests/benchmarks/conftest.py` — no real Cursor storage dependency.
+
+## CI gate
+
+The `benchmarks` job on **ubuntu-latest** runs the full `tests/benchmarks/` suite (`--benchmark-json=benchmark-results.json`), then `scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json`.
+
+- **Fail** when a gated mean exceeds its baseline by **>20%**
+- **Fail** when a gated mean is **<50%** of baseline (stale — refresh after intentional speedups)
+- **Fail** when a gated baseline name has no current result
+- **Warn** for benchmarks without a baseline entry
+- **Skip gate** for `EXCLUDED_FROM_GATE` names (smallest parse corpus, full-corpus search — sub-ms CI noise)
+
+Pinned runner: `ubuntu-latest`, `--benchmark-min-rounds=5`.
+
+## Refresh baselines
+
+After intentional performance work, capture on **ubuntu-latest** (same OS as the gated CI job). Download `benchmark-results.json` from a CI artifact when possible:
+
+```bash
+python scripts/reduce_baselines.py benchmark-results.json benchmarks/baselines.json --slack 1.5
+```
+
+For a quick local snapshot only (may not match CI timings):
+
+```bash
+make seed-baselines-local
+```
+
+`make update-baselines` is a deprecated alias for `seed-baselines-local`. Do not commit baselines from macOS/Windows unless you accept cross-OS gate skew.
+
+## Makefile targets
+
+| Target | Purpose |
+|--------|---------|
+| `make check-benchmarks` | Run suite + regression gate locally |
+| `make seed-baselines-local` | Capture local timings into `benchmarks/baselines.json` (with slack) |
+| `make clean-benchmark-artifacts` | Remove `benchmark-results.json` and `benchmarks/_raw.json` |

benchmarks/baselines.json

@@ -1,15 +1,29 @@
 {
-  "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR #120, run 28123677675). Refresh after intentional perf changes: download benchmark-results.json from the CI artifacts job, then `python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json` (re-seed with reduce_baselines or edit means). Local capture: `pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmark-results.json -o addopts=` on ubuntu-latest.",
-  "updated": "2026-06-24T19:20:27Z",
+  "_note": "Gated means from ubuntu-latest CI benchmark-results.json. Values multiplied by 1.5\u00d7 slack at generation time. Excluded from gate (recorded for reference): test_list_workspace_projects_nocache[composers-10], test_search_full_corpus. Refresh after intentional speedups via reduce_baselines.py.",
+  "updated": "2026-06-25T21:14:16Z",
   "machine": "Linux",
   "groups": {
+    "parse": {
+      "test_list_workspace_projects_nocache[composers-10]": 0.01702312019643009,
+      "test_list_workspace_projects_nocache[composers-50]": 0.07538331990000699,
+      "test_list_workspace_projects_nocache[composers-200]": 0.251991555999993
+    },
+    "export": {
+      "test_post_export_zip[composers-10]": 0.0112034034344294,
+      "test_post_export_zip[composers-50]": 0.04482855966665985
+    },
+    "search": {
+      "test_search_full_corpus": 0.047164217833331655
+    },
     "summary-cache": {
-      "test_summary_cache_hit": 6.3e-05,
-      "test_summary_cache_miss": 6.3e-05,
-      "test_fingerprint_workspace_entries[10]": 0.001844,
-      "test_fingerprint_workspace_entries[50]": 0.007759,
-      "test_fingerprint_workspace_entries[200]": 0.022231,
-      "test_summary_cache_round_trip": 0.000351
+      "test_summary_cache_lookup[hit]": 9.224067718099102e-05,
+      "test_summary_cache_lookup[miss]": 9.128770315496628e-05,
+      "test_fingerprint_workspace_entries[10]": 0.0024789120309553535,
+      "test_fingerprint_workspace_entries[50]": 0.010901568931818675,
+      "test_fingerprint_workspace_entries[200]": 0.03069810573000666,
+      "test_summary_cache_round_trip": 0.0004966099535917549,
+      "test_tab_summary_cache_lookup[hit]": 0.00010487297799045405,
+      "test_tab_summary_cache_lookup[miss]": 0.00010309520517204601
     }
   }
 }

scripts/check_benchmark_regression.py

@@ -8,6 +8,15 @@
 from pathlib import Path
 
 THRESHOLD = 1.20
+STALE_FLOOR = 0.50
+
+# Sub-ms timings are too noisy for a fixed 20% gate on ubuntu CI.
+EXCLUDED_FROM_GATE = frozenset(
+    {
+        "test_list_workspace_projects_nocache[composers-10]",
+        "test_search_full_corpus",
+    }
+)
 
 
 class BenchmarkDataError(ValueError):
@@ -102,14 +111,18 @@ def check_regression(
     baselines_path: str | Path,
     *,
     threshold: float = THRESHOLD,
+    stale_floor: float = STALE_FLOOR,
 ) -> int:
-    """Return 0 when within threshold; 1 when any gated benchmark regresses."""
+    """Return 0 when within threshold; 1 when any gated benchmark regresses or is stale."""
     flat = load_results(results_path)
     baseline_means = load_baseline_means(baselines_path)
 
     failures: list[str] = []
+    stale: list[str] = []
     missing: list[str] = []
     for name, base in baseline_means.items():
+        if name in EXCLUDED_FROM_GATE:
+            continue
         cur = flat.get(name)
         if cur is None:
             print(f"FAIL: no current result for gated baseline {name!r}")
@@ -119,20 +132,32 @@ def check_regression(
             print(f"WARN: baseline for {name!r} is zero; skipping ratio check")
             continue
         ratio = cur / base
-        tag = "FAIL" if ratio > threshold else "ok"
-        print(f"[{tag}] {name}: {cur:.6f}s vs {base:.6f}s ({ratio:.2f}x)")
         if ratio > threshold:
+            tag = "FAIL"
             failures.append(name)
+        elif ratio < stale_floor:
+            tag = "STALE"
+            stale.append(name)
+        else:
+            tag = "ok"
+        print(f"[{tag}] {name}: {cur:.6f}s vs {base:.6f}s ({ratio:.2f}x)")
 
     for name in flat:
+        if name in EXCLUDED_FROM_GATE:
+            continue
         if name not in baseline_means:
             print(f"WARN: {name!r} has no baseline yet; not gated")
 
     if failures:
         print(f"\nREGRESSION: {len(failures)} benchmark(s) exceeded {threshold:.0%}")
+    if stale:
+        print(
+            f"\nSTALE: {len(stale)} benchmark(s) are faster than {stale_floor:.0%} of baseline "
+            "(refresh baselines after intentional speedups)"
+        )
     if missing:
         print(f"\nMISSING: {len(missing)} gated benchmark(s) absent from current results")
-    if failures or missing:
+    if failures or stale or missing:
         return 1
     return 0
 
@@ -147,12 +172,25 @@ def main(argv: list[str] | None = None) -> int:
         default=THRESHOLD,
         help="fail when current mean exceeds baseline by more than this ratio (default: 1.20)",
     )
+    parser.add_argument(
+        "--stale-floor",
+        type=float,
+        default=STALE_FLOOR,
+        help="fail when current mean is below this fraction of baseline (default: 0.50)",
+    )
     args = parser.parse_args(argv)
+    if args.threshold <= 1:
+        print("ERROR: --threshold must be greater than 1", file=sys.stderr)
+        return 2
+    if not 0 < args.stale_floor < 1:
+        print("ERROR: --stale-floor must be between 0 and 1 (exclusive)", file=sys.stderr)
+        return 2
     try:
         return check_regression(
             args.results_path,
             args.baselines_path,
             threshold=args.threshold,
+            stale_floor=args.stale_floor,
         )
     except BenchmarkDataError as exc:
         print(f"ERROR: {exc}", file=sys.stderr)

scripts/reduce_baselines.py

@@ -0,0 +1,123 @@
+"""Reduce pytest-benchmark JSON into benchmarks/baselines.json."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from scripts.check_benchmark_regression import (
+    EXCLUDED_FROM_GATE,
+    BenchmarkDataError,
+    normalize_benchmark_name,
+)
+
+GATED_GROUPS = ("parse", "export", "search", "summary-cache")
+
+
+def _positive_float(value: str) -> float:
+    parsed = float(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("slack must be greater than zero")
+    return parsed
+
+
+def reduce_baselines(
+    raw_path: str | Path,
+    out_path: str | Path,
+    *,
+    slack: float = 1.0,
+    source: str = "local",
+) -> dict[str, object]:
+    path = Path(raw_path)
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise BenchmarkDataError(f"invalid JSON in {path}: {exc}") from exc
+    except OSError as exc:
+        raise BenchmarkDataError(f"cannot read {path}: {exc}") from exc
+
+    try:
+        entries = raw["benchmarks"]
+    except (KeyError, TypeError) as exc:
+        raise BenchmarkDataError(f"{path} missing top-level 'benchmarks' array") from exc
+    if not isinstance(entries, list):
+        raise BenchmarkDataError(f"{path} 'benchmarks' must be an array")
+
+    groups: dict[str, dict[str, float]] = {group: {} for group in GATED_GROUPS}
+    for index, entry in enumerate(entries):
+        if not isinstance(entry, dict):
+            raise BenchmarkDataError(f"{path} benchmarks[{index}] must be an object")
+        try:
+            raw_name = entry["name"]
+            mean = float(entry["stats"]["mean"])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise BenchmarkDataError(
+                f"{path} benchmarks[{index}] missing 'name' or 'stats.mean'"
+            ) from exc
+        bench_name = normalize_benchmark_name(str(raw_name))
+        group = entry.get("group")
+        if group not in GATED_GROUPS:
+            continue
+        groups[group][bench_name] = mean * slack
+
+    excluded = ", ".join(sorted(EXCLUDED_FROM_GATE))
+    slack_note = f" Values multiplied by {slack}× slack at generation time." if slack != 1.0 else ""
+    machine_info = raw.get("machine_info")
+    machine = machine_info.get("system") if isinstance(machine_info, dict) else None
+    source_labels = {
+        "ubuntu-latest-ci": "ubuntu-latest CI benchmark-results.json",
+        "local": "local benchmark-results.json",
+    }
+    source_label = source_labels.get(source, source)
+    output: dict[str, object] = {
+        "_note": (
+            f"Gated means from {source_label}."
+            f"{slack_note} "
+            f"Excluded from gate (recorded for reference): {excluded}. "
+            "Refresh after intentional speedups via reduce_baselines.py."
+        ),
+        "updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "machine": machine,
+        "groups": groups,
+    }
+    out = Path(out_path)
+    try:
+        out.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
+    except OSError as exc:
+        raise BenchmarkDataError(f"cannot write {out}: {exc}") from exc
+    return output
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("raw_path", help="pytest-benchmark --benchmark-json output")
+    parser.add_argument("out_path", help="destination baselines.json path")
+    parser.add_argument(
+        "--slack",
+        type=_positive_float,
+        default=1.0,
+        help="multiply means by this factor (must be > 0)",
+    )
+    parser.add_argument(
+        "--source",
+        default="local",
+        help="provenance label for _note (e.g. ubuntu-latest-ci, local)",
+    )
+    args = parser.parse_args(argv)
+    try:
+        reduce_baselines(args.raw_path, args.out_path, slack=args.slack, source=args.source)
+    except BenchmarkDataError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())