fix(bench): address bradjin8 review on search, cache, and baselines

Author: clean6378-max-it

.gitignore

@@ -46,4 +46,5 @@ coverage.xml
 .hypothesis/
 benchmark-results.json
 benchmarks/_raw.json
+benchmarks/_merged.json
 benchmarks/_ci/

Makefile

@@ -6,7 +6,9 @@
 seed-baselines-local:
 	@echo "WARNING: seed-baselines-local uses this host's timings; CI gates on ubuntu-latest." >&2
 	python -m pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts=
-	python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json --slack 1.5 --source local
+	python -c "import os, subprocess, sys; \
+	  cmd = [sys.executable, 'scripts/reduce_baselines.py', 'benchmarks/_raw.json', 'benchmarks/baselines.json', '--slack', '1.5', '--source', 'local']; \
+	  (subprocess.run(cmd, check=True), print('Updated benchmarks/baselines.json', file=sys.stderr)) if os.environ.get('FORCE') == '1' else print('Wrote benchmarks/_raw.json only. Set FORCE=1 to overwrite benchmarks/baselines.json.', file=sys.stderr)"
 
 # Deprecated alias — kept for muscle memory; see seed-baselines-local warning above.
 update-baselines: seed-baselines-local

benchmarks/README.md

@@ -17,12 +17,16 @@ pytest tests/benchmarks/ --benchmark-only -o addopts= -v
 | Group | What |
 |-------|------|
 | parse | `list_workspace_projects(..., nocache=True)` over 10 / 50 / 200 synthetic composers |
-| export | `POST /api/export` (ZIP) over 10 / 50 composer corpora |
-| search | `GET /api/search` over a 50-composer synthetic corpus |
-| summary-cache | cache lookup (hit/miss), fingerprint (10/50/200), round-trip, tab-summary lookup |
+| export | `POST /api/export` (ZIP) over 10 / 50 composer corpora (capped at 50 for CI runtime; parse goes to 200) |
+| search | `GET /api/search` over a 50-composer corpus — **live-scan** (`test_search_full_corpus_live_scan`, `NO_SEARCH_INDEX=1`) and **FTS index** (`test_search_full_corpus_indexed`, pre-built index) |
+| summary-cache | projects lookup (hit/miss), composer-map lookup (hit/miss), fingerprint (10/50/200), round-trip, tab-summary lookup |
 
 Synthetic corpora are built in `tests/benchmarks/conftest.py` — no real Cursor storage dependency.
 
+### Adding a benchmark group
+
+Every `@pytest.mark.benchmark(group="...")` name must appear in `GATED_GROUPS` inside `scripts/reduce_baselines.py`. Otherwise `reduce_baselines.py` fails at refresh time with an unknown-group error. Update both the test marker and `GATED_GROUPS` when introducing a new group.
+
 ## CI gate
 
 The `benchmarks` job on **ubuntu-latest** runs the full `tests/benchmarks/` suite (`--benchmark-json=benchmark-results.json`), then `scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json`.
@@ -35,18 +39,22 @@ The `benchmarks` job on **ubuntu-latest** runs the full `tests/benchmarks/` suit
 
 Pinned runner: `ubuntu-latest`, `--benchmark-min-rounds=5`.
 
+Sub-millisecond benches (e.g. `test_summary_cache_lookup`, `test_composer_map_cache_lookup`) can be high-variance on shared runners. If the gate becomes flaky, raise `--slack` for those entries or reintroduce targeted exclusions in `EXCLUDED_FROM_GATE`.
+
 ## Refresh baselines
 
 After intentional performance work, capture on **ubuntu-latest** (same OS as the gated CI job). Download `benchmark-results.json` from a CI artifact when possible:
 
 ```bash
-python scripts/reduce_baselines.py benchmark-results.json benchmarks/baselines.json --slack 1.5
+python scripts/reduce_baselines.py benchmark-results.json benchmarks/baselines.json --slack 1.5 --source ubuntu-latest-ci
 ```
 
 For a quick local snapshot only (may not match CI timings):
 
 ```bash
 make seed-baselines-local
+# writes benchmarks/_raw.json only; does not overwrite benchmarks/baselines.json
+make seed-baselines-local FORCE=1   # also runs reduce_baselines into benchmarks/baselines.json
 ```
 
 `make update-baselines` is a deprecated alias for `seed-baselines-local`. Do not commit baselines from macOS/Windows unless you accept cross-OS gate skew.
@@ -56,5 +64,5 @@ make seed-baselines-local
 | Target | Purpose |
 |--------|---------|
 | `make check-benchmarks` | Run suite + regression gate locally |
-| `make seed-baselines-local` | Capture local timings into `benchmarks/baselines.json` (with slack) |
+| `make seed-baselines-local` | Capture local timings to `benchmarks/_raw.json` (use `FORCE=1` to update `baselines.json`) |
 | `make clean-benchmark-artifacts` | Remove `benchmark-results.json` and `benchmarks/_raw.json` |

benchmarks/baselines.json

@@ -1,6 +1,6 @@
 {
   "_note": "Gated means from ubuntu-latest CI benchmark-results.json. Values multiplied by 1.5x slack at generation time. Refresh after intentional speedups via reduce_baselines.py.",
-  "updated": "2026-06-25T21:48:35Z",
+  "updated": "2026-06-25T22:57:33Z",
   "machine": "Linux",
   "groups": {
     "parse": {
@@ -13,7 +13,8 @@
       "test_post_export_zip[composers-50]": 0.04482855966665985
     },
     "search": {
-      "test_search_full_corpus": 0.047164217833331655
+      "test_search_full_corpus_live_scan": 0.047164217833331655,
+      "test_search_full_corpus_indexed": 0.05494209932945618
     },
     "summary-cache": {
       "test_summary_cache_lookup[hit]": 9.224067718099102e-05,
@@ -23,7 +24,9 @@
       "test_fingerprint_workspace_entries[200]": 0.03069810573000666,
       "test_summary_cache_round_trip": 0.0004966099535917549,
       "test_tab_summary_cache_lookup[hit]": 0.00010487297799045405,
-      "test_tab_summary_cache_lookup[miss]": 0.00010309520517204601
+      "test_tab_summary_cache_lookup[miss]": 0.00010309520517204601,
+      "test_composer_map_cache_lookup[hit]": 8.074544668606364e-05,
+      "test_composer_map_cache_lookup[miss]": 9.495690246481993e-05
     }
   }
 }

tests/benchmarks/conftest.py

@@ -122,11 +122,21 @@ def _make_bench_flask_client(
     monkeypatch: pytest.MonkeyPatch,
     *,
     state_subdir: str = ".cursor-chat-browser",
+    live_scan_search: bool = False,
 ) -> FlaskClient:
-    """Flask test client with env + export state patched for synthetic storage."""
+    """Flask test client with env + export state patched for synthetic storage.
+
+    When *live_scan_search* is True, set ``CURSOR_CHAT_BROWSER_NO_SEARCH_INDEX=1`` so
+    ``/api/search`` measures the live-scan fallback. Otherwise the FTS index path
+    from #113 may be used when an index is built (see indexed search fixtures).
+    """
     monkeypatch.setenv("WORKSPACE_PATH", storage["workspace_path"])
     monkeypatch.setenv("CLI_CHATS_PATH", storage["cli_chats_path"])
-    monkeypatch.setenv("CURSOR_CHAT_BROWSER_NO_SEARCH_INDEX", "1")
+    if live_scan_search:
+        monkeypatch.setenv("CURSOR_CHAT_BROWSER_NO_SEARCH_INDEX", "1")
+    else:
+        monkeypatch.delenv("CURSOR_CHAT_BROWSER_NO_SEARCH_INDEX", raising=False)
+        monkeypatch.delenv("CURSOR_CHAT_BROWSER_NOCACHE", raising=False)
     state_dir = tmp_path / state_subdir
     state_dir.mkdir()
     monkeypatch.setattr("api.export_api._get_state_dir", lambda: str(state_dir))
@@ -138,7 +148,12 @@ def _make_bench_flask_client(
 
 @pytest.fixture
 def summary_cache_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
-    """Redirect summary-cache files to an isolated temp directory."""
+    """Redirect summary-cache files to an isolated temp directory.
+
+    Tab-summary files use ``CACHE_DIR`` + hashed filenames only (see
+    ``summary_cache._tab_summaries_path``); they do not use
+    ``PROJECTS_CACHE_FILE`` or ``COMPOSER_MAP_CACHE_FILE``.
+    """
     cache_dir = tmp_path / "cache"
     cache_dir.mkdir()
     monkeypatch.setattr(summary_cache, "CACHE_DIR", cache_dir)
@@ -212,19 +227,54 @@ def bench_env(
 @pytest.fixture
 def bench_client(bench_env: dict[str, str], tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FlaskClient:
     """Flask test client bound to synthetic bench storage."""
-    return _make_bench_flask_client(bench_env, tmp_path, monkeypatch)
+    return _make_bench_flask_client(bench_env, tmp_path, monkeypatch, live_scan_search=True)
 
 
 @pytest.fixture
 def bench_client_search_corpus(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
 ) -> FlaskClient:
-    """Flask client over a fixed 50-composer corpus for search benchmarks."""
+    """Flask client over a fixed 50-composer corpus (live-scan search path)."""
     storage = build_bench_storage(tmp_path / "search_storage", 50)
     return _make_bench_flask_client(
         storage,
         tmp_path,
         monkeypatch,
         state_subdir=".cursor-chat-browser-search",
+        live_scan_search=True,
+    )
+
+
+@pytest.fixture
+def bench_client_search_corpus_indexed(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> FlaskClient:
+    """Flask client with FTS index built for the 50-composer search corpus."""
+    from services.search_index import build_search_index
+
+    monkeypatch.delenv("CURSOR_CHAT_BROWSER_NO_SEARCH_INDEX", raising=False)
+    monkeypatch.delenv("CURSOR_CHAT_BROWSER_NOCACHE", raising=False)
+
+    storage = build_bench_storage(tmp_path / "search_indexed_storage", 50)
+    cache_dir = tmp_path / "search_index_cache"
+    cache_dir.mkdir()
+    monkeypatch.setattr("services.search_index.CACHE_DIR", cache_dir)
+    monkeypatch.setattr(
+        "services.search_index.SEARCH_INDEX_POINTER_FILE",
+        cache_dir / "search_index.active",
+    )
+    monkeypatch.setattr(
+        "services.search_index.SEARCH_INDEX_FILE",
+        cache_dir / "search_index.sqlite",
+    )
+    built = build_search_index(storage["workspace_path"], [], force=True)
+    assert built is True
+    return _make_bench_flask_client(
+        storage,
+        tmp_path,
+        monkeypatch,
+        state_subdir=".cursor-chat-browser-search-indexed",
+        live_scan_search=False,
     )

tests/benchmarks/test_search_bench.py

@@ -8,19 +8,41 @@
 from tests.benchmarks.constants import BENCH_SEARCH_TERM
 
 
+def _search_url() -> str:
+    return f"/api/search?q={BENCH_SEARCH_TERM}&all_history=1"
+
+
+def _assert_search_response(response: object) -> None:
+    assert response.status_code == 200  # type: ignore[attr-defined]
+    body = response.get_json()  # type: ignore[attr-defined]
+    assert isinstance(body, dict)
+    results = body.get("results")
+    assert isinstance(results, list) and len(results) > 0
+
+
 @pytest.mark.benchmark(group="search")
-def test_search_full_corpus(
+def test_search_full_corpus_live_scan(
     benchmark,
     bench_client_search_corpus: FlaskClient,
 ) -> None:
+    """Live-scan fallback only (``CURSOR_CHAT_BROWSER_NO_SEARCH_INDEX=1``)."""
+
     def _run() -> object:
-        return bench_client_search_corpus.get(
-            f"/api/search?q={BENCH_SEARCH_TERM}&all_history=1",
-        )
+        return bench_client_search_corpus.get(_search_url())
 
     response = benchmark(_run)
-    assert response.status_code == 200
-    body = response.get_json()
-    assert isinstance(body, dict)
-    results = body.get("results")
-    assert isinstance(results, list) and len(results) > 0
+    _assert_search_response(response)
+
+
+@pytest.mark.benchmark(group="search")
+def test_search_full_corpus_indexed(
+    benchmark,
+    bench_client_search_corpus_indexed: FlaskClient,
+) -> None:
+    """FTS index path (#113) with pre-built ``search_index.sqlite``."""
+
+    def _run() -> object:
+        return bench_client_search_corpus_indexed.get(_search_url())
+
+    response = benchmark(_run)
+    _assert_search_response(response)

tests/benchmarks/test_summary_cache_bench.py

@@ -9,8 +9,10 @@
 
 from services.summary_cache import (
     fingerprint_workspace_storage,
+    get_cached_composer_id_to_ws,
     get_cached_projects,
     get_cached_tab_summaries,
+    set_cached_composer_id_to_ws,
     set_cached_projects,
     set_cached_tab_summaries,
 )
@@ -39,6 +41,26 @@ def test_summary_cache_lookup(
         assert result is None
 
 
+@pytest.mark.benchmark(group="summary-cache")
+@pytest.mark.parametrize("mode", ["hit", "miss"], ids=["hit", "miss"])
+def test_composer_map_cache_lookup(
+    benchmark,
+    mode: Literal["hit", "miss"],
+    summary_cache_dir: Path,
+    workspace_fingerprint: dict[str, Any],
+    stale_fingerprint: dict[str, Any],
+) -> None:
+    """Time ``get_cached_composer_id_to_ws`` hit/miss (fingerprint mismatch on miss)."""
+    mapping = {"cmp_0000": "ws_0000"}
+    set_cached_composer_id_to_ws(workspace_fingerprint, mapping)
+    lookup_fp = workspace_fingerprint if mode == "hit" else stale_fingerprint
+    result = benchmark(get_cached_composer_id_to_ws, lookup_fp)
+    if mode == "hit":
+        assert result == mapping
+    else:
+        assert result is None
+
+
 @pytest.mark.benchmark(group="summary-cache")
 @pytest.mark.parametrize(
     "synthetic_workspace",