Skip to content

Commit 31ac7e2

Browse files
clean6378-max-itclean6378-max-it
committed
Speed up workspace listing and search; add FTS index and fix search deep links
1 parent ee6a6e7 commit 31ac7e2

25 files changed

Lines changed: 2312 additions & 389 deletions

‎api/search.py‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"""
22
API route for search — mirrors src/app/api/search/route.ts
3-
GET /api/search?q=...&type=all|chat|composer
3+
GET /api/search?q=...&type=all|chat|composer&all_history=1
44
"""
55

66
import logging
@@ -12,7 +12,9 @@
1212

1313
from models import ParseWarningCollector, SearchResult
1414
from services.search import (
15+
DEFAULT_SEARCH_WINDOW_DAYS,
1516
rank_results,
17+
resolve_search_since_ms,
1618
search_cli_sessions,
1719
search_global_storage,
1820
search_legacy_workspaces,
@@ -23,12 +25,26 @@
2325
_logger = logging.getLogger(__name__)
2426

2527

28+
def _parse_since_days_param(raw: str | None) -> int | None:
29+
if raw is None or not str(raw).strip():
30+
return None
31+
try:
32+
return int(raw)
33+
except ValueError:
34+
return None
35+
36+
2637
@bp.route("/api/search")
2738
def search() -> tuple[Response, int] | Response:
2839
try:
2940
query = request.args.get("q", "").strip()
3041
search_type = request.args.get("type", "all")
3142
rules = current_app.config.get("EXCLUSION_RULES") or []
43+
all_history = request.args.get("all_history") in ("1", "true")
44+
since_ms = resolve_search_since_ms(
45+
all_history=all_history,
46+
since_days=_parse_since_days_param(request.args.get("since_days")),
47+
)
3248

3349
if not query:
3450
return json_response({"error": "No search query provided"}, 400)
@@ -40,20 +56,46 @@ def search() -> tuple[Response, int] | Response:
4056
if search_type != "chat":
4157
results.extend(
4258
search_global_storage(
43-
workspace_path, query, query_lower, rules, parse_warnings
59+
workspace_path,
60+
query,
61+
query_lower,
62+
rules,
63+
parse_warnings,
64+
since_ms=since_ms,
4465
)
4566
)
4667
results.extend(
47-
search_legacy_workspaces(workspace_path, query, query_lower, search_type, rules)
68+
search_legacy_workspaces(
69+
workspace_path,
70+
query,
71+
query_lower,
72+
search_type,
73+
rules,
74+
since_ms=since_ms,
75+
)
4876
)
4977
if search_type == "all":
5078
results.extend(
5179
search_cli_sessions(
52-
get_cli_chats_path(), query, query_lower, rules, parse_warnings
80+
get_cli_chats_path(),
81+
query,
82+
query_lower,
83+
rules,
84+
parse_warnings,
85+
since_ms=since_ms,
5386
)
5487
)
5588

56-
payload: dict[str, Any] = {"results": rank_results(results)}
89+
payload: dict[str, Any] = {
90+
"results": rank_results(results),
91+
"allHistory": since_ms is None,
92+
"searchWindowDays": (
93+
None if since_ms is None else (
94+
_parse_since_days_param(request.args.get("since_days"))
95+
or DEFAULT_SEARCH_WINDOW_DAYS
96+
)
97+
),
98+
}
5799
return json_response(parse_warnings.attach_to(payload))
58100

59101
except Exception:

‎app.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,17 @@ def inject_year() -> dict[str, int]:
6969
app.register_blueprint(pdf_bp)
7070
app.register_blueprint(config_bp)
7171

72+
try:
73+
from services.search_index import start_search_index_background
74+
from utils.workspace_path import resolve_workspace_path
75+
76+
start_search_index_background(
77+
resolve_workspace_path(),
78+
app.config["EXCLUSION_RULES"],
79+
)
80+
except Exception:
81+
logging.getLogger(__name__).exception("Failed to start search index background worker")
82+
7283
# ---------- Page routes ----------
7384

7485
@app.route("/")

‎scripts/profile_search.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
"""One-off search profiler — run: python scripts/profile_search.py [query]"""
2+
from __future__ import annotations
3+
4+
import os
5+
import sys
6+
import time
7+
8+
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
9+
if REPO_ROOT not in sys.path:
10+
sys.path.insert(0, REPO_ROOT)
11+
12+
from models import ParseWarningCollector
13+
from services.search import search_global_storage
14+
from services.workspace_db import (
15+
COMPOSER_ROWS_WITH_HEADERS_SQL,
16+
build_composer_id_to_workspace_id_cached,
17+
collect_workspace_entries,
18+
open_global_db,
19+
)
20+
from services.search import _index_bubble_texts_matching_query, _sql_like_substring
21+
from utils.workspace_path import resolve_workspace_path
22+
23+
24+
def main() -> None:
25+
query = sys.argv[1] if len(sys.argv) > 1 else "export"
26+
q = query.lower()
27+
wp = resolve_workspace_path()
28+
entries = collect_workspace_entries(wp)
29+
30+
t0 = time.perf_counter()
31+
build_composer_id_to_workspace_id_cached(wp, entries, [])
32+
print(f"composer_map: {time.perf_counter() - t0:.2f}s")
33+
34+
with open_global_db(wp) as (conn, _):
35+
if conn is None:
36+
print("no global db")
37+
return
38+
t0 = time.perf_counter()
39+
rows = conn.execute(COMPOSER_ROWS_WITH_HEADERS_SQL).fetchall()
40+
print(f"composer_rows {len(rows)}: {time.perf_counter() - t0:.2f}s")
41+
42+
t0 = time.perf_counter()
43+
bubble_count = conn.execute(
44+
"SELECT count(*) FROM cursorDiskKV WHERE key LIKE 'bubbleId:%'"
45+
).fetchone()[0]
46+
print(f"bubble_count {bubble_count}: {time.perf_counter() - t0:.2f}s")
47+
48+
t0 = time.perf_counter()
49+
idx = _index_bubble_texts_matching_query(conn, q)
50+
print(f"bubble_index {len(idx)} composers: {time.perf_counter() - t0:.2f}s")
51+
52+
pattern = _sql_like_substring(q)
53+
t0 = time.perf_counter()
54+
raw_hits = sum(
55+
1 for row in rows
56+
if q in (row["value"] or "").lower()
57+
)
58+
print(f"composer_raw_hits {raw_hits}: {time.perf_counter() - t0:.2f}s")
59+
60+
t0 = time.perf_counter()
61+
results = search_global_storage(wp, query, q, [], ParseWarningCollector())
62+
print(f"search_global_storage {len(results)} hits: {time.perf_counter() - t0:.2f}s")
63+
64+
65+
if __name__ == "__main__":
66+
main()

‎scripts/profile_search_breakdown.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
"""Break down where search time goes with vs without window."""
2+
from __future__ import annotations
3+
4+
import json
5+
import os
6+
import sys
7+
import time
8+
9+
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
10+
if REPO_ROOT not in sys.path:
11+
sys.path.insert(0, REPO_ROOT)
12+
13+
from services.search import (
14+
_composer_dict_timestamp_ms,
15+
_composer_row_raw_text,
16+
_index_bubble_texts_matching_query,
17+
_timestamp_in_search_window,
18+
resolve_search_since_ms,
19+
)
20+
from services.workspace_db import (
21+
COMPOSER_ROWS_WITH_HEADERS_SQL,
22+
build_composer_id_to_workspace_id_cached,
23+
collect_workspace_entries,
24+
open_global_db,
25+
)
26+
from utils.workspace_path import resolve_workspace_path
27+
28+
29+
def profile_window(since_ms, label: str) -> None:
30+
query = "export"
31+
q = query.lower()
32+
wp = resolve_workspace_path()
33+
entries = collect_workspace_entries(wp)
34+
35+
t0 = time.perf_counter()
36+
build_composer_id_to_workspace_id_cached(wp, entries, [])
37+
print(f" composer_map: {time.perf_counter() - t0:.2f}s")
38+
39+
with open_global_db(wp) as (conn, _):
40+
if conn is None:
41+
return
42+
t0 = time.perf_counter()
43+
composer_rows = conn.execute(COMPOSER_ROWS_WITH_HEADERS_SQL).fetchall()
44+
print(f" load_composer_rows ({len(composer_rows)}): {time.perf_counter() - t0:.2f}s")
45+
46+
window_ids = None
47+
if since_ms is not None:
48+
t0 = time.perf_counter()
49+
window_ids = set()
50+
in_window_rows = []
51+
for row in composer_rows:
52+
composer_id = row["key"].split(":")[1]
53+
raw_text = _composer_row_raw_text(row)
54+
try:
55+
cd_probe = json.loads(raw_text)
56+
except Exception:
57+
continue
58+
if not isinstance(cd_probe, dict):
59+
continue
60+
if not _timestamp_in_search_window(
61+
_composer_dict_timestamp_ms(cd_probe), since_ms
62+
):
63+
continue
64+
window_ids.add(composer_id)
65+
in_window_rows.append(row)
66+
composer_rows = in_window_rows
67+
print(
68+
f" window_filter ({len(window_ids)} composers): "
69+
f"{time.perf_counter() - t0:.2f}s"
70+
)
71+
72+
t0 = time.perf_counter()
73+
idx = _index_bubble_texts_matching_query(conn, q, composer_ids=window_ids)
74+
mode = "scoped" if window_ids is not None and len(window_ids) <= 500 else "full_scan"
75+
print(
76+
f" bubble_index ({len(idx)} hits, {mode}): "
77+
f"{time.perf_counter() - t0:.2f}s"
78+
)
79+
80+
t0 = time.perf_counter()
81+
hits = 0
82+
for row in composer_rows:
83+
raw = _composer_row_raw_text(row)
84+
cid = row["key"].split(":")[1]
85+
if q in raw.lower() or cid in idx:
86+
hits += 1
87+
print(f" composer_match_loop ({hits} candidates): {time.perf_counter() - t0:.2f}s")
88+
89+
90+
def main() -> None:
91+
since = resolve_search_since_ms(all_history=False)
92+
print("=== 30-day window ===")
93+
profile_window(since, "30d")
94+
print("=== all history ===")
95+
profile_window(None, "all")
96+
97+
98+
if __name__ == "__main__":
99+
main()

‎scripts/profile_search_window.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
"""Profile search with 30-day window vs all history."""
2+
from __future__ import annotations
3+
4+
import json
5+
import os
6+
import sys
7+
import time
8+
from datetime import datetime, timezone
9+
10+
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
11+
if REPO_ROOT not in sys.path:
12+
sys.path.insert(0, REPO_ROOT)
13+
14+
from models import ParseWarningCollector
15+
from services.search import (
16+
_composer_dict_timestamp_ms,
17+
_timestamp_in_search_window,
18+
resolve_search_since_ms,
19+
search_cli_sessions,
20+
search_global_storage,
21+
search_legacy_workspaces,
22+
)
23+
from services.workspace_db import COMPOSER_ROWS_WITH_HEADERS_SQL, open_global_db
24+
from utils.workspace_path import get_cli_chats_path, resolve_workspace_path
25+
26+
27+
def main() -> None:
28+
query = sys.argv[1] if len(sys.argv) > 1 else "export"
29+
q = query.lower()
30+
wp = resolve_workspace_path()
31+
since = resolve_search_since_ms(all_history=False)
32+
print(
33+
f"since_ms={since} "
34+
f"({datetime.fromtimestamp(since / 1000, tz=timezone.utc).isoformat()})"
35+
)
36+
37+
with open_global_db(wp) as (conn, _):
38+
if conn is None:
39+
print("no global db")
40+
return
41+
rows = conn.execute(COMPOSER_ROWS_WITH_HEADERS_SQL).fetchall()
42+
total = len(rows)
43+
in_window = 0
44+
unknown_ts = 0
45+
for row in rows:
46+
try:
47+
cd = json.loads(row["value"])
48+
except Exception:
49+
continue
50+
ts = _composer_dict_timestamp_ms(cd)
51+
if ts <= 0:
52+
unknown_ts += 1
53+
if _timestamp_in_search_window(ts, since):
54+
in_window += 1
55+
bubble_count = conn.execute(
56+
"SELECT count(*) FROM cursorDiskKV WHERE key LIKE 'bubbleId:%'"
57+
).fetchone()[0]
58+
59+
print(f"composers total={total} in_30d_window={in_window} unknown_ts={unknown_ts}")
60+
print(f"bubble_rows={bubble_count}")
61+
print(f"scoped_bubble_scan={in_window <= 500}")
62+
63+
pw = ParseWarningCollector()
64+
for label, since_ms in [("30d", since), ("all", None)]:
65+
t0 = time.perf_counter()
66+
r1 = search_global_storage(wp, query, q, [], pw, since_ms=since_ms)
67+
t1 = time.perf_counter() - t0
68+
t0 = time.perf_counter()
69+
r2 = search_legacy_workspaces(wp, query, q, "all", [], since_ms=since_ms)
70+
t2 = time.perf_counter() - t0
71+
t0 = time.perf_counter()
72+
r3 = search_cli_sessions(get_cli_chats_path(), query, q, [], pw, since_ms=since_ms)
73+
t3 = time.perf_counter() - t0
74+
total_hits = len(r1) + len(r2) + len(r3)
75+
total_s = t1 + t2 + t3
76+
print(
77+
f"{label}: global={len(r1)} ({t1:.2f}s) "
78+
f"legacy={len(r2)} ({t2:.2f}s) cli={len(r3)} ({t3:.2f}s) "
79+
f"total={total_hits} ({total_s:.2f}s)"
80+
)
81+
82+
83+
if __name__ == "__main__":
84+
main()

0 commit comments

Comments
 (0)