Add exact totals to search pagination

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/api/v2/routers/search_router.py

@@ -65,7 +65,6 @@ async def search(
         has_filters=bool(query.note_types or query.entity_types or query.metadata_filters),
     ):
         offset = (page - 1) * page_size
-        fetch_limit = page_size + 1
         try:
             with logfire.span(
                 "api.search.search.execute_query",
@@ -75,7 +74,8 @@ async def search(
                 page=page,
                 page_size=page_size,
             ):
-                results = await search_service.search(query, limit=fetch_limit, offset=offset)
+                results = await search_service.search(query, limit=page_size, offset=offset)
+                total = await search_service.count(query)
         except SemanticSearchDisabledError as exc:
             raise HTTPException(status_code=400, detail=str(exc)) from exc
         except SemanticDependenciesMissingError as exc:
@@ -90,9 +90,7 @@ async def search(
             phase="paginate_results",
             result_count=len(results),
         ):
-            has_more = len(results) > page_size
-            if has_more:
-                results = results[:page_size]
+            has_more = offset + len(results) < total
 
         with logfire.span(
             "api.search.search.hydrate_results",
@@ -113,6 +111,7 @@ async def search(
                 results=search_results,
                 current_page=page,
                 page_size=page_size,
+                total=total,
                 has_more=has_more,
             )
 

src/basic_memory/repository/postgres_search_repository.py

@@ -686,7 +686,17 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non
     # FTS search (Postgres-specific)
     # ------------------------------------------------------------------
 
-    async def search(
+    @staticmethod
+    def _is_tsquery_syntax_error(exc: Exception) -> bool:
+        msg = str(exc).lower()
+        return (
+            "syntax error in tsquery" in msg
+            or "invalid input syntax for type tsquery" in msg
+            or "no operand in tsquery" in msg
+            or "no operator in tsquery" in msg
+        )
+
+    async def _build_fts_query_parts(
         self,
         search_text: Optional[str] = None,
         permalink: Optional[str] = None,
@@ -696,31 +706,8 @@ async def search(
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
         metadata_filters: Optional[dict] = None,
-        retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
-        min_similarity: Optional[float] = None,
-        limit: int = 10,
-        offset: int = 0,
-    ) -> List[SearchIndexRow]:
-        """Search across all indexed content using PostgreSQL tsvector."""
-        # --- Dispatch vector / hybrid modes (shared logic) ---
-        dispatched = await self._dispatch_retrieval_mode(
-            search_text=search_text,
-            permalink=permalink,
-            permalink_match=permalink_match,
-            title=title,
-            note_types=note_types,
-            after_date=after_date,
-            search_item_types=search_item_types,
-            metadata_filters=metadata_filters,
-            retrieval_mode=retrieval_mode,
-            min_similarity=min_similarity,
-            limit=limit,
-            offset=offset,
-        )
-        if dispatched is not None:
-            return dispatched
-
-        # --- FTS mode (Postgres-specific) ---
+    ) -> tuple[str, str, dict, str, str]:
+        """Build Postgres FTS FROM/WHERE params shared by search and count."""
         conditions = []
         params = {}
         order_by_clause = ""
@@ -868,10 +855,6 @@ async def search(
         params["project_id"] = self.project_id
         conditions.append("search_index.project_id = :project_id")
 
-        # set limit and offset
-        params["limit"] = limit
-        params["offset"] = offset
-
         # Build WHERE clause
         where_clause = " AND ".join(conditions) if conditions else "1=1"
 
@@ -884,6 +867,64 @@ async def search(
         else:
             score_expr = "0"
 
+        return from_clause, where_clause, params, order_by_clause, score_expr
+
+    async def search(
+        self,
+        search_text: Optional[str] = None,
+        permalink: Optional[str] = None,
+        permalink_match: Optional[str] = None,
+        title: Optional[str] = None,
+        note_types: Optional[List[str]] = None,
+        after_date: Optional[datetime] = None,
+        search_item_types: Optional[List[SearchItemType]] = None,
+        metadata_filters: Optional[dict] = None,
+        retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
+        min_similarity: Optional[float] = None,
+        limit: int = 10,
+        offset: int = 0,
+    ) -> List[SearchIndexRow]:
+        """Search across all indexed content using PostgreSQL tsvector."""
+        # --- Dispatch vector / hybrid modes (shared logic) ---
+        dispatched = await self._dispatch_retrieval_mode(
+            search_text=search_text,
+            permalink=permalink,
+            permalink_match=permalink_match,
+            title=title,
+            note_types=note_types,
+            after_date=after_date,
+            search_item_types=search_item_types,
+            metadata_filters=metadata_filters,
+            retrieval_mode=retrieval_mode,
+            min_similarity=min_similarity,
+            limit=limit,
+            offset=offset,
+        )
+        if dispatched is not None:
+            return dispatched
+
+        # --- FTS mode (Postgres-specific) ---
+        (
+            from_clause,
+            where_clause,
+            params,
+            order_by_clause,
+            score_expr,
+        ) = await self._build_fts_query_parts(
+            search_text=search_text,
+            permalink=permalink,
+            permalink_match=permalink_match,
+            title=title,
+            note_types=note_types,
+            after_date=after_date,
+            search_item_types=search_item_types,
+            metadata_filters=metadata_filters,
+        )
+
+        # set limit and offset
+        params["limit"] = limit
+        params["offset"] = offset
+
         sql = f"""
             SELECT
                 search_index.project_id,
@@ -915,17 +956,7 @@ async def search(
                 result = await session.execute(text(sql), params)
                 rows = result.fetchall()
         except Exception as e:
-            # Handle tsquery syntax errors (and only those).
-            #
-            # Important: Postgres errors for other failures (e.g. missing table) will still mention
-            # `to_tsquery(...)` in the SQL text, so checking for the substring "tsquery" is too broad.
-            msg = str(e).lower()
-            if (
-                "syntax error in tsquery" in msg
-                or "invalid input syntax for type tsquery" in msg
-                or "no operand in tsquery" in msg
-                or "no operator in tsquery" in msg
-            ):
+            if self._is_tsquery_syntax_error(e):
                 logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}")
                 return []
 
@@ -966,3 +997,65 @@ async def search(
             )
 
         return results
+
+    async def count(
+        self,
+        search_text: Optional[str] = None,
+        permalink: Optional[str] = None,
+        permalink_match: Optional[str] = None,
+        title: Optional[str] = None,
+        note_types: Optional[List[str]] = None,
+        after_date: Optional[datetime] = None,
+        search_item_types: Optional[List[SearchItemType]] = None,
+        metadata_filters: Optional[dict] = None,
+        retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
+        min_similarity: Optional[float] = None,
+    ) -> int:
+        """Count indexed content matching the Postgres FTS query."""
+        mode = (
+            retrieval_mode.value
+            if isinstance(retrieval_mode, SearchRetrievalMode)
+            else str(retrieval_mode)
+        )
+        if mode != SearchRetrievalMode.FTS.value:
+            return await super().count(
+                search_text=search_text,
+                permalink=permalink,
+                permalink_match=permalink_match,
+                title=title,
+                note_types=note_types,
+                after_date=after_date,
+                search_item_types=search_item_types,
+                metadata_filters=metadata_filters,
+                retrieval_mode=retrieval_mode,
+                min_similarity=min_similarity,
+            )
+
+        (
+            from_clause,
+            where_clause,
+            params,
+            _order_by_clause,
+            _score_expr,
+        ) = await self._build_fts_query_parts(
+            search_text=search_text,
+            permalink=permalink,
+            permalink_match=permalink_match,
+            title=title,
+            note_types=note_types,
+            after_date=after_date,
+            search_item_types=search_item_types,
+            metadata_filters=metadata_filters,
+        )
+        sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
+        logger.trace(f"Count {sql} params: {params}")
+        try:
+            async with db.scoped_session(self.session_maker) as session:
+                result = await session.execute(text(sql), params)
+                return int(result.scalar_one())
+        except Exception as e:
+            if self._is_tsquery_syntax_error(e):
+                logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}")
+                return 0
+            logger.error(f"Database error during search count: {e}")
+            raise

src/basic_memory/repository/search_repository.py

@@ -50,6 +50,22 @@ async def search(
         """Search across indexed content."""
         ...
 
+    async def count(
+        self,
+        search_text: Optional[str] = None,
+        permalink: Optional[str] = None,
+        permalink_match: Optional[str] = None,
+        title: Optional[str] = None,
+        note_types: Optional[List[str]] = None,
+        after_date: Optional[datetime] = None,
+        search_item_types: Optional[List[SearchItemType]] = None,
+        metadata_filters: Optional[dict] = None,
+        retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
+        min_similarity: Optional[float] = None,
+    ) -> int:
+        """Count indexed content matching the same filters as search."""
+        ...
+
     async def index_item(self, search_index_row: SearchIndexRow) -> None:
         """Index a single item."""
         ...

src/basic_memory/repository/search_repository_base.py

@@ -247,6 +247,36 @@ async def search(
         """
         pass
 
+    async def count(
+        self,
+        search_text: Optional[str] = None,
+        permalink: Optional[str] = None,
+        permalink_match: Optional[str] = None,
+        title: Optional[str] = None,
+        note_types: Optional[List[str]] = None,
+        after_date: Optional[datetime] = None,
+        search_item_types: Optional[List[SearchItemType]] = None,
+        metadata_filters: Optional[Dict[str, Any]] = None,
+        retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
+        min_similarity: Optional[float] = None,
+    ) -> int:
+        """Count results for retrieval modes that cannot use a backend COUNT query."""
+        results = await self.search(
+            search_text=search_text,
+            permalink=permalink,
+            permalink_match=permalink_match,
+            title=title,
+            note_types=note_types,
+            after_date=after_date,
+            search_item_types=search_item_types,
+            metadata_filters=metadata_filters,
+            retrieval_mode=retrieval_mode,
+            min_similarity=min_similarity,
+            limit=VECTOR_FILTER_SCAN_LIMIT,
+            offset=0,
+        )
+        return len(results)
+
     # ------------------------------------------------------------------
     # Abstract methods — semantic search (backend-specific DB operations)
     # ------------------------------------------------------------------