Fixed Timon's feedback on feat/search-fts-index. Summary: utils/search_index.py Added ORDER BY m.timestamp_ms DESC so index hits are newest-first Replaced the 20× over-fetch with a fixed _FTS_BATCH_SIZE = 200; _search_via_index keeps paging until results are filled or FTS is exhausted Fixed offset: sql_rows_fetched is now rows actually scanned (not full batch size), so early break no longer skips unprocessed rows api/search.py Added _rank_search_hits() for timestamp-desc ordering Index and live-scan paths both rank before returning (live scan collects all matches, then sorts and slices — correct global newest-first on fallback) Tests test_results_ordered_newest_first — verifies DESC ordering test_phrase_filter_paginates_past_decoy_token_matches — 250 token-only decoys + real phrase at end; would fail with the old offset bug 30 tests passed (ruff + mypy clean). Suggested commit message: fix: FTS search ordering and pagination for phrase filtering Add ORDER BY timestamp_ms DESC, advance SQL offset by rows scanned, and page in fixed batches until enough hits or FTS exhausted. Rank live-scan results the same way for parity. Want me to commit and push to PR #120? /humanizer commit message fix: FTS search ordering and pagination

Author: clean6378-max-it

api/search.py

@@ -20,6 +20,7 @@
     resolve_search_since_ms,
     search_snippet,
     timestamp_in_search_window_iso,
+    timestamp_to_ms,
     tool_result_searchable_text,
 )
 from utils.session_cache import get_cached_session
@@ -59,6 +60,17 @@ def _parse_since_days(raw: str | None) -> int | None:
     return min(days, _MAX_SEARCH_SINCE_DAYS)
 
 
+def _rank_search_hits(results: list[SearchHitDict]) -> list[SearchHitDict]:
+    """Sort hits by timestamp descending (missing timestamps last)."""
+    return sorted(
+        results,
+        key=lambda hit: timestamp_to_ms(
+            hit["timestamp"] if isinstance(hit.get("timestamp"), str) else None
+        ),
+        reverse=True,
+    )
+
+
 def _message_searchable_text(msg: MessageDict) -> str:
     text = msg.get("text", "") or msg.get("content", "")
     if not isinstance(text, str):
@@ -149,7 +161,7 @@ def _search_via_index(
         sql_offset += indexed["sql_rows_fetched"]
         if indexed["sql_exhausted"]:
             break
-    return results
+    return _rank_search_hits(results)[:max_results]
 
 
 def _search_live_scan(
@@ -164,12 +176,8 @@ def _search_live_scan(
     projects = list_projects(base)
     results: list[SearchHitDict] = []
     for project in projects:
-        if len(results) >= max_results:
-            break
         sessions = list_sessions(project["path"])
         for sess_info in sessions:
-            if len(results) >= max_results:
-                break
             try:
                 session = get_cached_session(sess_info["path"])
             except Exception:
@@ -202,9 +210,7 @@ def _search_live_scan(
                         "snippet": search_snippet(text, query),
                     }
                 )
-                if len(results) >= max_results:
-                    break
-    return results
+    return _rank_search_hits(results)[:max_results]
 
 
 @search_bp.route("/api/search")

tests/test_search.py

@@ -147,7 +147,12 @@ def test_search_falls_back_when_index_query_fails(tmp_path, monkeypatch):
     client = _seed_indexed_client(tmp_path, monkeypatch, timestamp=recent_ts)
     with patch(
         "api.search.query_index_hits",
-        return_value={"hits": [], "query_ok": False},
+        return_value={
+            "hits": [],
+            "query_ok": False,
+            "sql_rows_fetched": 0,
+            "sql_exhausted": True,
+        },
     ):
         resp = client.get("/api/search?q=Hello")
     assert resp.status_code == 200

tests/test_search_index.py

@@ -11,6 +11,7 @@
 
 import pytest
 
+from api.search import _search_via_index
 from utils.search_index import (
     build_search_index,
     index_is_usable,
@@ -238,6 +239,93 @@ def test_undated_message_in_window(self, tmp_path, monkeypatch):
 
 
 class TestQueryIndexHits:
+    def test_results_ordered_newest_first(self, tmp_path, monkeypatch):
+        cache_root = tmp_path / "cache"
+        cache_root.mkdir()
+        projects = tmp_path / "projects"
+        _write_session(
+            projects / "order-proj" / "session.jsonl",
+            [
+                {
+                    "type": "user",
+                    "timestamp": "2026-05-19T10:00:00Z",
+                    "message": {"content": [{"type": "text", "text": "order alpha token"}]},
+                },
+                {
+                    "type": "user",
+                    "timestamp": "2026-06-19T10:00:00Z",
+                    "message": {"content": [{"type": "text", "text": "order beta token"}]},
+                },
+                {
+                    "type": "user",
+                    "timestamp": "2026-07-19T10:00:00Z",
+                    "message": {"content": [{"type": "text", "text": "order gamma token"}]},
+                },
+            ],
+        )
+        monkeypatch.setenv("CLAUDE_CODE_CHAT_BROWSER_SEARCH_INDEX_DIR", str(cache_root))
+        patches = _index_patches(cache_root)
+        with patches[0]:
+            build_search_index(str(projects), [], force=True)
+            hits = query_index_hits("order", since_ms=None, max_results=10)
+            assert hits["query_ok"] is True
+            assert len(hits["hits"]) == 3
+            timestamps = [hit["timestamp"] for hit in hits["hits"]]
+            assert timestamps == sorted(timestamps, reverse=True)
+
+    def test_phrase_filter_paginates_past_decoy_token_matches(self, tmp_path, monkeypatch):
+        """Multi-word phrase matches beyond one FTS batch are not skipped."""
+        cache_root = tmp_path / "cache"
+        cache_root.mkdir()
+        projects = tmp_path / "projects"
+        lines: list[dict[str, object]] = []
+        for i in range(250):
+            lines.append(
+                {
+                    "type": "user",
+                    "timestamp": f"2026-07-{(i % 28) + 1:02d}T10:00:00Z",
+                    "message": {
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": f"phrase target decoy unique token only {i}",
+                            }
+                        ]
+                    },
+                }
+            )
+        phrase = "decoy unique token only phrase target"
+        lines.append(
+            {
+                "type": "user",
+                "timestamp": "2025-12-01T10:00:00Z",
+                "message": {
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": phrase,
+                        }
+                    ]
+                },
+            }
+        )
+        _write_session(projects / "page-proj" / "session.jsonl", lines)
+        monkeypatch.setenv("CLAUDE_CODE_CHAT_BROWSER_SEARCH_INDEX_DIR", str(cache_root))
+        patches = _index_patches(cache_root)
+        with patches[0]:
+            build_search_index(str(projects), [], force=True)
+            results = _search_via_index(
+                str(projects),
+                [],
+                phrase,
+                phrase.lower(),
+                since_ms=None,
+                max_results=1,
+            )
+            assert results is not None
+            assert len(results) == 1
+            assert phrase in results[0]["snippet"] or "phrase target" in results[0]["snippet"]
+
     def test_fts_failure_returns_query_not_ok(self, indexed_tree):
         @contextmanager
         def _broken_conn(*, readonly: bool = True):

utils/search_index.py

@@ -58,6 +58,7 @@
 _usability_cache: dict[tuple[str, str], tuple[bool, float]] = {}
 _usability_cache_lock = threading.Lock()
 _USABILITY_CACHE_TTL_SECONDS = 30.0
+_FTS_BATCH_SIZE = 200
 
 
 class IndexMessageHitDict(TypedDict):
@@ -628,7 +629,7 @@ def query_index_hits(
     if not fts_q:
         return empty
 
-    sql_limit = max(max_results * 20, max_results)
+    sql_limit = max(max_results, _FTS_BATCH_SIZE)
     with _index_db_conn(readonly=True) as conn:
         if conn is None:
             return {"hits": [], "query_ok": False, "sql_rows_fetched": 0, "sql_exhausted": True}
@@ -640,6 +641,7 @@ def query_index_hits(
                 " JOIN sessions s"
                 " ON s.session_id = m.session_id AND s.project_name = m.project_name"
                 " WHERE messages_fts MATCH ?"
+                " ORDER BY m.timestamp_ms DESC"
                 " LIMIT ? OFFSET ?",
                 (fts_q, sql_limit, sql_offset),
             ).fetchall()
@@ -648,7 +650,9 @@ def query_index_hits(
             return {"hits": [], "query_ok": False, "sql_rows_fetched": 0, "sql_exhausted": True}
 
     hits: list[IndexMessageHitDict] = []
+    rows_scanned = 0
     for row in rows:
+        rows_scanned += 1
         text = row["text"] or ""
         if query_lower not in text.lower():
             continue
@@ -672,7 +676,7 @@ def query_index_hits(
     return {
         "hits": hits,
         "query_ok": True,
-        "sql_rows_fetched": len(rows),
+        "sql_rows_fetched": rows_scanned,
         "sql_exhausted": len(rows) < sql_limit,
     }