fix(core): purge SQLite search_index on project delete (#832)

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/alembic/versions/n7i8j9k0l1m2_cleanup_sqlite_search_orphans.py

@@ -0,0 +1,86 @@
+"""Remove orphaned search rows whose project was already deleted.
+
+Revision ID: n7i8j9k0l1m2
+Revises: m6h7i8j9k0l1
+Create Date: 2026-05-15 18:30:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+from sqlalchemy import inspect
+
+
+revision: str = "n7i8j9k0l1m2"
+down_revision: Union[str, None] = "m6h7i8j9k0l1"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _table_exists(connection, table_name: str) -> bool:
+    """Inspector-based table check, dialect agnostic.
+
+    Trigger: SQLite creates search_index as an FTS5 virtual table at runtime
+    via SearchRepository.init_search_index, not through Alembic, so fresh
+    installs hit this migration before the table exists.
+    Why: a blind DELETE against a missing table fails the whole upgrade.
+    Outcome: callers skip the sweep when the table isn't present yet — the
+    runtime-created table on a fresh DB has no orphans to clean.
+    """
+    return table_name in inspect(connection).get_table_names()
+
+
+def upgrade() -> None:
+    """Purge orphaned search rows left over from prior project deletions.
+
+    Trigger: project deletion on SQLite never removed the derived FTS rows,
+    because the FTS5 virtual table can't carry a foreign key. The leak shows
+    up in two shapes:
+    1. project_id no longer exists in `project` (deleted project, id never
+       reused).
+    2. project_id still exists but `entity_id` no longer exists in `entity`
+       — auto-increment handed the id to a brand-new project and the FTS
+       rows from the deleted predecessor masquerade as the new tenant's data.
+    Why: search_index.project_id is the only scope predicate the search
+    repository applies, so leftover rows surface under the wrong project on
+    every search.
+    Outcome: a one-time sweep deletes both shapes, from the FTS index and
+    from search_vector_chunks. Postgres already cascaded on FK delete, so
+    these statements are no-ops there.
+    """
+    connection = op.get_bind()
+
+    if _table_exists(connection, "search_index"):
+        op.execute(
+            """
+            DELETE FROM search_index
+            WHERE project_id NOT IN (SELECT id FROM project)
+            """
+        )
+        op.execute(
+            """
+            DELETE FROM search_index
+            WHERE entity_id IS NOT NULL
+              AND entity_id NOT IN (SELECT id FROM entity)
+            """
+        )
+
+    if _table_exists(connection, "search_vector_chunks"):
+        op.execute(
+            """
+            DELETE FROM search_vector_chunks
+            WHERE project_id NOT IN (SELECT id FROM project)
+            """
+        )
+        op.execute(
+            """
+            DELETE FROM search_vector_chunks
+            WHERE entity_id NOT IN (SELECT id FROM entity)
+            """
+        )
+
+
+def downgrade() -> None:
+    """No-op: orphan rows cannot be reconstructed."""
+    pass

src/basic_memory/mcp/tools/search.py

@@ -10,8 +10,13 @@
 from fastmcp import Context
 from pydantic import AliasChoices, BeforeValidator, Field
 
-from basic_memory.config import ConfigManager
+from basic_memory.config import ConfigManager, has_cloud_credentials
 from basic_memory.utils import build_canonical_permalink, coerce_dict, coerce_list
+from basic_memory.mcp.async_client import (
+    _explicit_routing,
+    _force_local_mode,
+    is_factory_mode,
+)
 from basic_memory.mcp.container import get_container
 from basic_memory.mcp.project_context import (
     detect_project_from_identifier_prefix,
@@ -479,12 +484,30 @@ async def _search_all_projects(
     total = 0
     any_project_has_more = False
 
+    # Trigger: caller asked for an account-wide search.
+    # Why: project_id (external UUID) routes through the cloud v2 API path,
+    #      which 401s on local installs because there's no JWT to present.
+    #      Project names route through the local-ASGI path and work for both
+    #      backends — cloud disambiguates names via the workspace/project
+    #      qualified_name already baked into project_ref["project"].
+    # Outcome: forward project_id only when the same signals get_project_client
+    #          uses to pick a cloud route are present. Mirrors the cloud_available
+    #          composite in project_context.get_project_client (single source of
+    #          truth for "can we route to cloud?").
+    config = ConfigManager().config
+    use_cloud_routing = (
+        is_factory_mode()
+        or (_explicit_routing() and not _force_local_mode())
+        or has_cloud_credentials(config)
+    )
+
     for project_ref in project_refs:
+        recursive_project_id = project_ref["project_id"] if use_cloud_routing else None
         try:
             results = await search_notes(
                 query=query,
                 project=project_ref["project"],
-                project_id=project_ref["project_id"],
+                project_id=recursive_project_id,
                 page=1,
                 page_size=per_project_page_size,
                 search_type=search_type,

src/basic_memory/repository/project_repository.py

@@ -4,14 +4,75 @@
 from typing import Optional, Sequence, Union
 
 
-from sqlalchemy import text
+from loguru import logger
+from sqlalchemy import inspect as sa_inspect, select, text
+from sqlalchemy.exc import NoResultFound, OperationalError
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
 from basic_memory import db
 from basic_memory.models.project import Project
 from basic_memory.repository.repository import Repository
 
 
+async def _load_sqlite_vec_on_session(session) -> bool:
+    """Ensure the sqlite-vec extension is loaded on this session's connection.
+
+    Returns True when vec0 is available after the call. Returns False when the
+    extension can't be loaded on this Python build (e.g., python.org macOS or
+    Windows interpreters without `enable_load_extension`) — every connection in
+    the pool shares the same interpreter, so a False here also means no
+    embedding row could ever have been written, and skipping the embeddings
+    purge is safe.
+
+    Mirrors SQLiteSearchRepository._ensure_sqlite_vec_loaded but as a free
+    function: we don't have a SearchRepository instance during project delete,
+    and the per-connection nature of extension loading means a pooled connection
+    routed to this session might not have vec loaded even when other
+    connections wrote embeddings.
+    """
+    try:
+        await session.execute(text("SELECT vec_version()"))
+        return True
+    except OperationalError:
+        pass
+
+    try:
+        import sqlite_vec  # type: ignore[import-not-found]
+    except ImportError:
+        logger.debug("sqlite-vec package not installed; skipping vec purge")
+        return False
+
+    async_connection = await session.connection()
+    raw_connection = await async_connection.get_raw_connection()
+    driver_connection = raw_connection.driver_connection
+
+    if not hasattr(driver_connection, "enable_load_extension"):
+        # Trigger: CPython build without sqlite extension support (#711).
+        # Why: load_extension is unavailable, so no connection in this pool
+        #      can host vec0. No embeddings exist anywhere.
+        # Outcome: skip the embeddings purge entirely.
+        logger.debug(
+            "Skipping search_vector_embeddings purge: this Python build does "
+            "not support SQLite extension loading"
+        )
+        return False
+
+    try:
+        await driver_connection.enable_load_extension(True)
+        await driver_connection.load_extension(sqlite_vec.loadable_path())
+        await driver_connection.enable_load_extension(False)
+        await session.execute(text("SELECT vec_version()"))
+    except Exception as exc:
+        logger.warning(
+            "Failed to load sqlite-vec for project delete cleanup; "
+            "skipping embeddings purge: {}",
+            exc,
+        )
+        return False
+
+    return True
+
+
 class ProjectRepository(Repository[Project]):
     """Repository for Project model.
 
@@ -121,6 +182,83 @@ async def set_as_default(self, project_id: int) -> Optional[Project]:
                 return target_project
             return None  # pragma: no cover
 
+    async def delete(self, entity_id: int) -> bool:
+        """Delete a project and its derived search rows in one transaction.
+
+        The cascade picture differs by backend:
+
+        - search_index → project: Postgres has ON DELETE CASCADE FK; SQLite
+          stores search_index as an FTS5 virtual table and can't carry FKs,
+          so it needs explicit cleanup.
+        - search_vector_chunks → project: neither backend has an FK here, so
+          both need an explicit DELETE.
+        - search_vector_embeddings → search_vector_chunks: Postgres has an FK
+          (chunk_id REFERENCES … ON DELETE CASCADE); SQLite stores embeddings
+          in a vec0 virtual table keyed by rowid with no cascade. On SQLite
+          the embeddings must be purged before the chunk rows, otherwise
+          `_run_vector_query` keeps returning stale vectors that crowd out
+          live results.
+
+        Each derived table is created lazily (search_index by
+        SearchRepository.init_search_index, the vector tables once semantic
+        search initializes), so any of them may be absent on minimal test DBs.
+        Inspect the connection once and skip whichever is missing.
+        """
+        logger.debug(f"Deleting Project and search rows for project_id: {entity_id}")
+        async with db.scoped_session(self.session_maker) as session:
+            try:
+                result = await session.execute(
+                    select(self.Model).filter(self.primary_key == entity_id)
+                )
+                project = result.scalars().one()
+            except NoResultFound:
+                logger.debug(f"No Project found to delete: {entity_id}")
+                return False
+
+            dialect_name = session.bind.dialect.name if session.bind else "sqlite"
+            is_sqlite = dialect_name == "sqlite"
+
+            existing_tables = await session.run_sync(
+                lambda sync_session: set(sa_inspect(sync_session.connection()).get_table_names())
+            )
+
+            # search_index: SQLite has no FK on the FTS5 virtual table; Postgres
+            # cascades from the project FK, so the explicit DELETE is redundant.
+            if is_sqlite and "search_index" in existing_tables:
+                await session.execute(
+                    text("DELETE FROM search_index WHERE project_id = :project_id"),
+                    {"project_id": entity_id},
+                )
+
+            # search_vector_chunks: no FK to project on either backend, so both
+            # backends need this. SQLite must purge vec0 embeddings first
+            # (rowid pseudocolumn — Postgres uses chunk_id and would 500 here);
+            # Postgres' chunk_id FK CASCADE handles its embeddings cleanup when
+            # we delete the chunk rows below.
+            if "search_vector_chunks" in existing_tables:
+                if is_sqlite and "search_vector_embeddings" in existing_tables:
+                    # Extension loading is per-connection. We must load vec0 on
+                    # *this* session before the DELETE; otherwise a different
+                    # pooled connection might have written embeddings that we'd
+                    # silently leave behind.
+                    if await _load_sqlite_vec_on_session(session):
+                        await session.execute(
+                            text(
+                                "DELETE FROM search_vector_embeddings WHERE rowid IN ("
+                                "SELECT id FROM search_vector_chunks "
+                                "WHERE project_id = :project_id)"
+                            ),
+                            {"project_id": entity_id},
+                        )
+                await session.execute(
+                    text("DELETE FROM search_vector_chunks WHERE project_id = :project_id"),
+                    {"project_id": entity_id},
+                )
+
+            await session.delete(project)
+            logger.debug(f"Deleted Project and search rows for project_id: {entity_id}")
+            return True
+
     async def update_path(self, project_id: int, new_path: str) -> Optional[Project]:
         """Update project path.