fix(bench): gate all baseline benchmarks and validate finite ratios

Author: clean6378-max-it

.github/workflows/tests.yml

@@ -115,7 +115,7 @@ jobs:
         # exercise Flask routes via app.test_client(). Only listed files — not
         # `pytest tests/` — to avoid re-collecting unittest.TestCase classes above.
         # -o addopts= avoids inheriting benchmark-only options from pyproject.toml.
-        run: python -m pytest tests/test_api_search.py tests/test_api_workspaces.py tests/test_api_export.py tests/test_pdf_export.py tests/test_search_helpers.py tests/test_check_benchmark_regression.py -v --tb=short -o addopts=
+        run: python -m pytest tests/test_api_search.py tests/test_api_workspaces.py tests/test_api_export.py tests/test_pdf_export.py tests/test_search_helpers.py tests/test_check_benchmark_regression.py tests/test_reduce_baselines.py -v --tb=short -o addopts=
 
       # ── PyInstaller desktop build (Windows only, once per workflow) ────────
       # Closes #44. Builds the onedir bundle and smoke-tests --help so the

Makefile

@@ -6,7 +6,7 @@
 seed-baselines-local:
 	@echo "WARNING: seed-baselines-local uses this host's timings; CI gates on ubuntu-latest." >&2
 	python -m pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
-	python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json --slack 1.5
+	python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json --slack 1.5 --source local
 
 # Deprecated alias — kept for muscle memory; see seed-baselines-local warning above.
 update-baselines: seed-baselines-local
@@ -16,4 +16,4 @@ check-benchmarks:
 	python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json
 
 clean-benchmark-artifacts:
-	rm -f benchmarks/_raw.json benchmark-results.json
+	python -c "import pathlib; [p.unlink(missing_ok=True) for p in (pathlib.Path('benchmarks/_raw.json'), pathlib.Path('benchmark-results.json'))]"

benchmarks/README.md

@@ -31,7 +31,7 @@ The `benchmarks` job on **ubuntu-latest** runs the full `tests/benchmarks/` suit
 - **Fail** when a gated mean is **<50%** of baseline (stale — refresh after intentional speedups)
 - **Fail** when a gated baseline name has no current result
 - **Warn** for benchmarks without a baseline entry
-- **Skip gate** for `EXCLUDED_FROM_GATE` names (smallest parse corpus, full-corpus search — sub-ms CI noise)
+- All benchmarks listed in `baselines.json` are gated (no exclusion list)
 
 Pinned runner: `ubuntu-latest`, `--benchmark-min-rounds=5`.
 

benchmarks/baselines.json

@@ -1,6 +1,6 @@
 {
-  "_note": "Gated means from ubuntu-latest CI benchmark-results.json. Values multiplied by 1.5\u00d7 slack at generation time. Excluded from gate (recorded for reference): test_list_workspace_projects_nocache[composers-10], test_search_full_corpus. Refresh after intentional speedups via reduce_baselines.py.",
-  "updated": "2026-06-25T21:14:16Z",
+  "_note": "Gated means from ubuntu-latest CI benchmark-results.json. Values multiplied by 1.5\u00d7 slack at generation time. Refresh after intentional speedups via reduce_baselines.py.",
+  "updated": "2026-06-25T21:48:35Z",
   "machine": "Linux",
   "groups": {
     "parse": {

scripts/check_benchmark_regression.py

@@ -4,19 +4,15 @@
 
 import argparse
 import json
+import math
 import sys
 from pathlib import Path
 
 THRESHOLD = 1.20
 STALE_FLOOR = 0.50
 
-# Sub-ms timings are too noisy for a fixed 20% gate on ubuntu CI.
-EXCLUDED_FROM_GATE = frozenset(
-    {
-        "test_list_workspace_projects_nocache[composers-10]",
-        "test_search_full_corpus",
-    }
-)
+# Benchmarks gated via baselines.json; empty set means all baseline entries are checked.
+EXCLUDED_FROM_GATE: frozenset[str] = frozenset()
 
 
 class BenchmarkDataError(ValueError):
@@ -106,6 +102,17 @@ def load_baseline_means(baselines_path: str | Path) -> dict[str, float]:
     return means
 
 
+def _validate_gate_ratios(threshold: float, stale_floor: float) -> None:
+    if not math.isfinite(threshold):
+        raise BenchmarkDataError("threshold must be finite")
+    if threshold <= 1:
+        raise BenchmarkDataError("threshold must be greater than 1")
+    if not math.isfinite(stale_floor):
+        raise BenchmarkDataError("stale_floor must be finite")
+    if not 0 < stale_floor < 1:
+        raise BenchmarkDataError("stale_floor must be between 0 and 1 (exclusive)")
+
+
 def check_regression(
     results_path: str | Path,
     baselines_path: str | Path,
@@ -114,6 +121,7 @@ def check_regression(
     stale_floor: float = STALE_FLOOR,
 ) -> int:
     """Return 0 when within threshold; 1 when any gated benchmark regresses or is stale."""
+    _validate_gate_ratios(threshold, stale_floor)
     flat = load_results(results_path)
     baseline_means = load_baseline_means(baselines_path)
 
@@ -179,12 +187,6 @@ def main(argv: list[str] | None = None) -> int:
         help="fail when current mean is below this fraction of baseline (default: 0.50)",
     )
     args = parser.parse_args(argv)
-    if args.threshold <= 1:
-        print("ERROR: --threshold must be greater than 1", file=sys.stderr)
-        return 2
-    if not 0 < args.stale_floor < 1:
-        print("ERROR: --stale-floor must be between 0 and 1 (exclusive)", file=sys.stderr)
-        return 2
     try:
         return check_regression(
             args.results_path,

scripts/reduce_baselines.py

@@ -4,6 +4,7 @@
 
 import argparse
 import json
+import math
 import sys
 from datetime import UTC, datetime
 from pathlib import Path
@@ -23,6 +24,8 @@
 
 def _positive_float(value: str) -> float:
     parsed = float(value)
+    if not math.isfinite(parsed):
+        raise argparse.ArgumentTypeError("slack must be a finite number")
     if parsed <= 0:
         raise argparse.ArgumentTypeError("slack must be greater than zero")
     return parsed
@@ -63,11 +66,23 @@ def reduce_baselines(
             ) from exc
         bench_name = normalize_benchmark_name(str(raw_name))
         group = entry.get("group")
+        if group is None:
+            raise BenchmarkDataError(
+                f"{path} benchmarks[{index}] ({bench_name!r}) missing required 'group'"
+            )
         if group not in GATED_GROUPS:
-            continue
+            raise BenchmarkDataError(
+                f"{path} benchmarks[{index}] ({bench_name!r}) has unknown group {group!r}; "
+                f"expected one of {GATED_GROUPS}"
+            )
         groups[group][bench_name] = mean * slack
 
     excluded = ", ".join(sorted(EXCLUDED_FROM_GATE))
+    excluded_note = (
+        f" Excluded from gate (recorded for reference): {excluded}."
+        if excluded
+        else ""
+    )
     slack_note = f" Values multiplied by {slack}× slack at generation time." if slack != 1.0 else ""
     machine_info = raw.get("machine_info")
     machine = machine_info.get("system") if isinstance(machine_info, dict) else None
@@ -79,8 +94,7 @@ def reduce_baselines(
     output: dict[str, object] = {
         "_note": (
             f"Gated means from {source_label}."
-            f"{slack_note} "
-            f"Excluded from gate (recorded for reference): {excluded}. "
+            f"{slack_note}{excluded_note} "
             "Refresh after intentional speedups via reduce_baselines.py."
         ),
         "updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),

tests/benchmarks/conftest.py

@@ -14,8 +14,7 @@
 from app import create_app
 from services import summary_cache
 from services.summary_cache import fingerprint_workspace_storage
-
-BENCH_SEARCH_TERM = "bench-search-token"
+from tests.benchmarks.constants import BENCH_SEARCH_TERM
 
 
 def make_workspace_entries(workspace_root: Path, count: int) -> list[dict[str, Any]]:
@@ -117,6 +116,26 @@ def build_bench_storage(root: Path, composer_count: int) -> dict[str, str]:
     }
 
 
+def _make_bench_flask_client(
+    storage: dict[str, str],
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    *,
+    state_subdir: str = ".cursor-chat-browser",
+) -> FlaskClient:
+    """Flask test client with env + export state patched for synthetic storage."""
+    monkeypatch.setenv("WORKSPACE_PATH", storage["workspace_path"])
+    monkeypatch.setenv("CLI_CHATS_PATH", storage["cli_chats_path"])
+    monkeypatch.setenv("CURSOR_CHAT_BROWSER_NO_SEARCH_INDEX", "1")
+    state_dir = tmp_path / state_subdir
+    state_dir.mkdir()
+    monkeypatch.setattr("api.export_api._get_state_dir", lambda: str(state_dir))
+    app = create_app()
+    app.config["TESTING"] = True
+    app.config["EXCLUSION_RULES"] = []
+    return app.test_client()
+
+
 @pytest.fixture
 def summary_cache_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
     """Redirect summary-cache files to an isolated temp directory."""
@@ -193,13 +212,7 @@ def bench_env(
 @pytest.fixture
 def bench_client(bench_env: dict[str, str], tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FlaskClient:
     """Flask test client bound to synthetic bench storage."""
-    state_dir = tmp_path / ".cursor-chat-browser"
-    state_dir.mkdir()
-    monkeypatch.setattr("api.export_api._get_state_dir", lambda: str(state_dir))
-    app = create_app()
-    app.config["TESTING"] = True
-    app.config["EXCLUSION_RULES"] = []
-    return app.test_client()
+    return _make_bench_flask_client(bench_env, tmp_path, monkeypatch)
 
 
 @pytest.fixture
@@ -209,13 +222,9 @@ def bench_client_search_corpus(
 ) -> FlaskClient:
     """Flask client over a fixed 50-composer corpus for search benchmarks."""
     storage = build_bench_storage(tmp_path / "search_storage", 50)
-    monkeypatch.setenv("WORKSPACE_PATH", storage["workspace_path"])
-    monkeypatch.setenv("CLI_CHATS_PATH", storage["cli_chats_path"])
-    monkeypatch.setenv("CURSOR_CHAT_BROWSER_NO_SEARCH_INDEX", "1")
-    state_dir = tmp_path / ".cursor-chat-browser-search"
-    state_dir.mkdir()
-    monkeypatch.setattr("api.export_api._get_state_dir", lambda: str(state_dir))
-    app = create_app()
-    app.config["TESTING"] = True
-    app.config["EXCLUSION_RULES"] = []
-    return app.test_client()
+    return _make_bench_flask_client(
+        storage,
+        tmp_path,
+        monkeypatch,
+        state_subdir=".cursor-chat-browser-search",
+    )

tests/benchmarks/constants.py

@@ -0,0 +1,3 @@
+"""Shared constants for benchmark corpora (importable outside conftest)."""
+
+BENCH_SEARCH_TERM = "bench-search-token"

tests/benchmarks/test_search_bench.py

@@ -5,7 +5,7 @@
 import pytest
 from flask.testing import FlaskClient
 
-from tests.benchmarks.conftest import BENCH_SEARCH_TERM
+from tests.benchmarks.constants import BENCH_SEARCH_TERM
 
 
 @pytest.mark.benchmark(group="search")

tests/benchmarks/test_summary_cache_bench.py

@@ -74,6 +74,11 @@ def _run() -> None:
         get_cached_projects(fp)
 
     benchmark(_run)
+    cached = get_cached_projects(fp)
+    assert cached is not None
+    cached_projects, cached_warnings = cached
+    assert cached_projects == projects
+    assert cached_warnings == []
 
 
 @pytest.mark.benchmark(group="summary-cache")