fix(core): address project index review bugs

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/index/local_watch.py

@@ -5,7 +5,7 @@
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Generic, Protocol, TypeVar
+from typing import Generic, Protocol, TypeVar, runtime_checkable
 
 from watchfiles.main import FileChange
 
@@ -38,6 +38,17 @@ class LocalWatchProjectSource(Protocol):
     def path(self) -> object: ...
 
 
+@runtime_checkable
+class LocalWatchProjectIdentitySource(Protocol):
+    """Optional stable project identity for local watcher storage prefixes."""
+
+    @property
+    def permalink(self) -> object | None: ...
+
+    @property
+    def name(self) -> object | None: ...
+
+
 def local_project_root(project: LocalWatchProjectSource) -> Path:
     """Return the resolved filesystem root for a local watcher project."""
     project_path = str(project.path).strip() if project.path else ""
@@ -48,6 +59,12 @@ def local_project_root(project: LocalWatchProjectSource) -> Path:
 
 def local_project_prefix(project: LocalWatchProjectSource) -> ProjectPath:
     """Return the storage-event prefix for a local watcher project."""
+    if isinstance(project, LocalWatchProjectIdentitySource):
+        for project_identity in (project.permalink, project.name):
+            project_prefix = str(project_identity).strip() if project_identity else ""
+            if project_prefix:
+                return project_prefix
+
     return local_project_root(project).name
 
 

src/basic_memory/indexing/project_index_workflow.py

@@ -30,6 +30,7 @@
     should_emit_project_index_progress_event,
 )
 from basic_memory.models import Entity, NoteContent, Relation
+from basic_memory.repository.project_repository import _load_sqlite_vec_on_session
 from basic_memory.runtime import (
     ProjectExternalId,
     ProjectId,
@@ -739,6 +740,40 @@ def missing_paths(self) -> tuple[str, ...]:
       AND entity_id IN :deleted_entity_ids
 """).bindparams(bindparam("deleted_entity_ids", expanding=True))
 
+SELECT_PROJECT_INDEX_SQLITE_VECTOR_TABLES_SQL = text("""
+    SELECT name
+    FROM sqlite_master
+    WHERE type = 'table'
+      AND name IN ('search_vector_chunks', 'search_vector_embeddings')
+""")
+
+SELECT_PROJECT_INDEX_POSTGRES_VECTOR_TABLES_SQL = text("""
+    SELECT table_name
+    FROM information_schema.tables
+    WHERE table_schema = ANY (current_schemas(false))
+      AND table_name IN ('search_vector_chunks', 'search_vector_embeddings')
+""")
+
+DELETE_PROJECT_INDEX_SQLITE_VECTOR_EMBEDDINGS_SQL = text("""
+    DELETE FROM search_vector_embeddings
+    WHERE rowid IN (
+        SELECT id
+        FROM search_vector_chunks
+        WHERE project_id = :project_id
+          AND entity_id IN :deleted_entity_ids
+    )
+""").bindparams(bindparam("deleted_entity_ids", expanding=True))
+
+DELETE_PROJECT_INDEX_POSTGRES_VECTOR_EMBEDDINGS_SQL = text("""
+    DELETE FROM search_vector_embeddings
+    WHERE chunk_id IN (
+        SELECT id
+        FROM search_vector_chunks
+        WHERE project_id = :project_id
+          AND entity_id IN :deleted_entity_ids
+    )
+""").bindparams(bindparam("deleted_entity_ids", expanding=True))
+
 PROJECT_INDEX_SEARCH_INDEX_TABLE = table(
     "search_index",
     column("project_id"),
@@ -749,6 +784,62 @@ def missing_paths(self) -> tuple[str, ...]:
 )
 
 
+def project_index_session_dialect_name(session: AsyncSession) -> str:
+    """Return the SQLAlchemy dialect name for project-index maintenance."""
+    return session.get_bind().dialect.name
+
+
+async def project_index_vector_table_names(session: AsyncSession) -> frozenset[str]:
+    """Return available vector table names for the current database backend."""
+    dialect_name = project_index_session_dialect_name(session)
+    if dialect_name == "sqlite":
+        result = await session.execute(SELECT_PROJECT_INDEX_SQLITE_VECTOR_TABLES_SQL)
+    elif dialect_name == "postgresql":
+        result = await session.execute(SELECT_PROJECT_INDEX_POSTGRES_VECTOR_TABLES_SQL)
+    else:
+        raise RuntimeError(f"Unsupported project-index database dialect: {dialect_name}")
+
+    return frozenset(str(table_name) for table_name in result.scalars())
+
+
+async def delete_project_index_vector_rows(
+    session: AsyncSession,
+    *,
+    project_id: ProjectId,
+    entity_ids: Sequence[int],
+) -> None:
+    """Delete backend vector rows for project-index entity deletes when tables exist."""
+    deleted_entity_ids = tuple(entity_ids)
+    if not deleted_entity_ids:
+        return
+
+    vector_table_names = await project_index_vector_table_names(session)
+    if "search_vector_chunks" not in vector_table_names:
+        return
+
+    delete_params = {
+        "project_id": project_id,
+        "deleted_entity_ids": deleted_entity_ids,
+    }
+    if "search_vector_embeddings" in vector_table_names:
+        dialect_name = project_index_session_dialect_name(session)
+        if dialect_name == "sqlite":
+            if await _load_sqlite_vec_on_session(session):
+                await session.execute(
+                    DELETE_PROJECT_INDEX_SQLITE_VECTOR_EMBEDDINGS_SQL,
+                    delete_params,
+                )
+        elif dialect_name == "postgresql":
+            await session.execute(
+                DELETE_PROJECT_INDEX_POSTGRES_VECTOR_EMBEDDINGS_SQL,
+                delete_params,
+            )
+        else:
+            raise RuntimeError(f"Unsupported project-index database dialect: {dialect_name}")
+
+    await session.execute(DELETE_PROJECT_INDEX_VECTOR_CHUNKS_SQL, delete_params)
+
+
 async def delete_project_index_entities(
     session: AsyncSession,
     *,
@@ -779,7 +870,11 @@ async def delete_project_index_entities(
         "relation_row_type": "relation",
     }
     await session.execute(DELETE_PROJECT_INDEX_SEARCH_ROWS_SQL, delete_params)
-    await session.execute(DELETE_PROJECT_INDEX_VECTOR_CHUNKS_SQL, delete_params)
+    await delete_project_index_vector_rows(
+        session,
+        project_id=project_id,
+        entity_ids=deleted_entity_ids,
+    )
     await session.execute(
         delete(Entity).where(
             Entity.project_id == project_id,

tests/index/test_local_watch_orchestration.py

@@ -159,6 +159,47 @@ def test_local_watch_request_builds_storage_prefix_from_project(tmp_path: Path)
     assert [event.object_key for event in source.events()] == ["configured-project-root/notes/a.md"]
 
 
+def test_local_watch_request_uses_project_permalink_for_duplicate_leaf_roots(
+    tmp_path: Path,
+) -> None:
+    alpha_root = tmp_path / "alpha" / "notes"
+    beta_root = tmp_path / "beta" / "notes"
+    alpha_note = alpha_root / "a.md"
+    beta_note = beta_root / "b.md"
+    alpha_note.parent.mkdir(parents=True)
+    beta_note.parent.mkdir(parents=True)
+    alpha_note.write_text("# A\n", encoding="utf-8")
+    beta_note.write_text("# B\n", encoding="utf-8")
+    alpha = SimpleNamespace(
+        path=str(alpha_root),
+        name="Alpha Notes",
+        permalink="alpha-notes",
+    )
+    beta = SimpleNamespace(
+        path=str(beta_root),
+        name="Beta Notes",
+        permalink="beta-notes",
+    )
+
+    alpha_request = LocalWatchEventIndexRequest.from_project_changes(
+        project=alpha,
+        changes=((Change.added, str(alpha_note)),),
+    )
+    beta_request = LocalWatchEventIndexRequest.from_project_changes(
+        project=beta,
+        changes=((Change.added, str(beta_note)),),
+    )
+
+    assert alpha_request.project_prefix == "alpha-notes"
+    assert beta_request.project_prefix == "beta-notes"
+    assert [event.object_key for event in LocalWatchStorageEventSource(alpha_request).events()] == [
+        "alpha-notes/a.md"
+    ]
+    assert [event.object_key for event in LocalWatchStorageEventSource(beta_request).events()] == [
+        "beta-notes/b.md"
+    ]
+
+
 def test_local_watch_project_change_batches_route_changes_to_projects(tmp_path: Path) -> None:
     alpha_root = tmp_path / "alpha"
     beta_root = tmp_path / "beta"

tests/indexing/test_project_index_workflow.py

@@ -3,6 +3,7 @@
 from collections.abc import AsyncIterator, Iterator
 from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
+from types import SimpleNamespace
 from typing import cast
 from uuid import UUID
 
@@ -154,9 +155,13 @@ class FakeProjectIndexSession:
     """Record repository maintenance statements without a real database."""
 
     results: list[FakeProjectIndexResult]
+    dialect_name: str = "sqlite"
     statements: list[object] = field(default_factory=list)
     params: list[object | None] = field(default_factory=list)
 
+    def get_bind(self) -> SimpleNamespace:
+        return SimpleNamespace(dialect=SimpleNamespace(name=self.dialect_name))
+
     async def execute(
         self,
         statement: object,
@@ -776,7 +781,7 @@ async def fake_scoped_session(
     assert "SELECT entity.id, entity.file_path" in str(session.statements[1])
     assert "SELECT DISTINCT relation.from_id" in str(session.statements[2])
     assert "DELETE FROM search_index" in str(session.statements[3])
-    assert "DELETE FROM search_vector_chunks" in str(session.statements[4])
+    assert "sqlite_master" in str(session.statements[4])
     assert "DELETE FROM entity" in str(session.statements[5])
     assert "UPDATE entity" in str(session.statements[6])
     assert "UPDATE note_content" in str(session.statements[7])
@@ -907,10 +912,133 @@ async def fake_scoped_session(
     assert "SELECT entity.id, entity.file_path" in str(session.statements[0])
     assert "SELECT DISTINCT relation.from_id" in str(session.statements[1])
     assert "DELETE FROM search_index" in str(session.statements[2])
-    assert "DELETE FROM search_vector_chunks" in str(session.statements[3])
+    assert "sqlite_master" in str(session.statements[3])
     assert "DELETE FROM entity" in str(session.statements[4])
 
 
+@pytest.mark.asyncio
+async def test_repository_project_index_maintenance_store_deletes_vector_embeddings_before_chunks(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    session_maker = cast(async_sessionmaker[AsyncSession], object())
+    session = FakeProjectIndexSession(
+        results=[
+            FakeProjectIndexResult(
+                mapping_rows=[
+                    {"id": 10, "file_path": "notes/a.md"},
+                ]
+            ),
+            FakeProjectIndexResult(scalar_values=[]),
+            FakeProjectIndexResult(),
+            FakeProjectIndexResult(
+                scalar_values=["search_vector_chunks", "search_vector_embeddings"]
+            ),
+        ]
+    )
+
+    @asynccontextmanager
+    async def fake_scoped_session(
+        scoped_session_maker: async_sessionmaker[AsyncSession],
+    ) -> AsyncIterator[FakeProjectIndexSession]:
+        assert scoped_session_maker is session_maker
+        yield session
+
+    monkeypatch.setattr(
+        project_index_workflow_module.db,
+        "scoped_session",
+        fake_scoped_session,
+    )
+    sqlite_vec_sessions: list[FakeProjectIndexSession] = []
+
+    async def fake_load_sqlite_vec_on_session(
+        loaded_session: FakeProjectIndexSession,
+    ) -> bool:
+        sqlite_vec_sessions.append(loaded_session)
+        return True
+
+    monkeypatch.setattr(
+        project_index_workflow_module,
+        "_load_sqlite_vec_on_session",
+        fake_load_sqlite_vec_on_session,
+    )
+
+    store = RepositoryProjectIndexMaintenanceStore(
+        session_maker=session_maker,
+        project_id=42,
+    )
+
+    result = await store.apply_project_index_delete_batch(
+        ProjectIndexDeleteBatch(
+            completed_batches=1,
+            paths=("notes/a.md",),
+        )
+    )
+
+    assert result.deleted_entities == 1
+    assert sqlite_vec_sessions == [session]
+    statements = [str(statement) for statement in session.statements]
+    embedding_delete_index = next(
+        index
+        for index, statement in enumerate(statements)
+        if "DELETE FROM search_vector_embeddings" in statement
+    )
+    chunk_delete_index = next(
+        index
+        for index, statement in enumerate(statements)
+        if "DELETE FROM search_vector_chunks" in statement
+    )
+    assert embedding_delete_index < chunk_delete_index
+
+
+@pytest.mark.asyncio
+async def test_repository_project_index_maintenance_store_skips_vector_cleanup_when_tables_missing(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    session_maker = cast(async_sessionmaker[AsyncSession], object())
+    session = FakeProjectIndexSession(
+        results=[
+            FakeProjectIndexResult(
+                mapping_rows=[
+                    {"id": 10, "file_path": "notes/a.md"},
+                ]
+            ),
+            FakeProjectIndexResult(scalar_values=[]),
+            FakeProjectIndexResult(),
+            FakeProjectIndexResult(scalar_values=[]),
+        ]
+    )
+
+    @asynccontextmanager
+    async def fake_scoped_session(
+        scoped_session_maker: async_sessionmaker[AsyncSession],
+    ) -> AsyncIterator[FakeProjectIndexSession]:
+        assert scoped_session_maker is session_maker
+        yield session
+
+    monkeypatch.setattr(
+        project_index_workflow_module.db,
+        "scoped_session",
+        fake_scoped_session,
+    )
+
+    store = RepositoryProjectIndexMaintenanceStore(
+        session_maker=session_maker,
+        project_id=42,
+    )
+
+    result = await store.apply_project_index_delete_batch(
+        ProjectIndexDeleteBatch(
+            completed_batches=1,
+            paths=("notes/a.md",),
+        )
+    )
+
+    assert result.deleted_entities == 1
+    statements = [str(statement) for statement in session.statements]
+    assert not any("DELETE FROM search_vector_chunks" in statement for statement in statements)
+    assert not any("DELETE FROM search_vector_embeddings" in statement for statement in statements)
+
+
 def test_project_index_workflow_start_builds_existing_metadata_and_attempt_event() -> None:
     tenant_id = UUID("11111111-1111-1111-1111-111111111111")
     workflow_id = UUID("22222222-2222-2222-2222-222222222222")