feat(core): support note-level embedding opt-out

phernandez · phernandez · commit cdffd879a9e8 · 2026-04-08T00:34:44.000-05:00
Signed-off-by: phernandez &lt;paul@basicmachines.co&gt;
diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py
@@ -427,6 +427,15 @@ async def index_entity_data(
 
     async def sync_entity_vectors(self, entity_id: int) -> None:
         """Refresh vector chunks for one entity in repositories that support semantic indexing."""
+        entity = await self.entity_repository.find_by_id(entity_id)
+        if entity is None:
+            await self._clear_entity_vectors(entity_id)
+            return
+
+        if not self._entity_embeddings_enabled(entity):
+            await self._clear_entity_vectors(entity_id)
+            return
+
         await self.repository.sync_entity_vectors(entity_id)
 
     async def sync_entity_vectors_batch(
@@ -435,10 +444,47 @@ async def sync_entity_vectors_batch(
         progress_callback=None,
     ) -> VectorSyncBatchResult:
         """Refresh vector chunks for a batch of entities."""
-        return await self.repository.sync_entity_vectors_batch(
-            entity_ids,
+        if not entity_ids:
+            return VectorSyncBatchResult(
+                entities_total=0,
+                entities_synced=0,
+                entities_failed=0,
+            )
+
+        entities_by_id = {
+            entity.id: entity for entity in await self.entity_repository.find_by_ids(entity_ids)
+        }
+        opted_out_ids = [
+            entity_id
+            for entity_id in entity_ids
+            if (
+                (entity := entities_by_id.get(entity_id)) is not None
+                and not self._entity_embeddings_enabled(entity)
+            )
+        ]
+        for entity_id in opted_out_ids:
+            await self._clear_entity_vectors(entity_id)
+
+        eligible_entity_ids = [
+            entity_id
+            for entity_id in entity_ids
+            if entity_id in entities_by_id and entity_id not in opted_out_ids
+        ]
+        if not eligible_entity_ids:
+            return VectorSyncBatchResult(
+                entities_total=len(entity_ids),
+                entities_synced=0,
+                entities_failed=0,
+                entities_skipped=len(opted_out_ids),
+            )
+
+        batch_result = await self.repository.sync_entity_vectors_batch(
+            eligible_entity_ids,
             progress_callback=progress_callback,
         )
+        batch_result.entities_total = len(entity_ids)
+        batch_result.entities_skipped += len(opted_out_ids)
+        return batch_result
 
     async def reindex_vectors(self, progress_callback=None) -> dict:
         """Rebuild vector embeddings for all entities.
@@ -463,7 +509,7 @@ async def reindex_vectors(self, progress_callback=None) -> dict:
         stats = {
             "total_entities": batch_result.entities_total,
             "embedded": batch_result.entities_synced,
-            "skipped": 0,
+            "skipped": batch_result.entities_skipped,
             "errors": batch_result.entities_failed,
         }
 
@@ -518,6 +564,60 @@ async def _purge_stale_search_rows(self) -> None:
 
         logger.info("Purged stale search rows for deleted entities", project_id=project_id)
 
+    @staticmethod
+    def _entity_embeddings_enabled(entity: Entity) -> bool:
+        """Return whether semantic embeddings should be generated for this entity."""
+        if not entity.entity_metadata:
+            return True
+
+        embed_value = entity.entity_metadata.get("embed")
+        if embed_value is None:
+            return True
+        if isinstance(embed_value, bool):
+            return embed_value
+        if isinstance(embed_value, str):
+            normalized = embed_value.strip().lower()
+            if normalized in {"false", "0", "no", "off"}:
+                return False
+            if normalized in {"true", "1", "yes", "on"}:
+                return True
+        if isinstance(embed_value, (int, float)):
+            return bool(embed_value)
+
+        # Default unknown values to enabled so malformed metadata does not silently
+        # remove notes from semantic search.
+        return True
+
+    async def _clear_entity_vectors(self, entity_id: int) -> None:
+        """Delete derived vector rows for one entity."""
+        from basic_memory.repository.search_repository_base import SearchRepositoryBase
+        from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
+
+        # Trigger: semantic indexing is disabled for this repository instance.
+        # Why: repositories only create vector tables when semantic search is enabled.
+        # Outcome: skip cleanup because there are no active derived vector rows to maintain.
+        if isinstance(self.repository, SearchRepositoryBase) and not self.repository._semantic_enabled:
+            return
+
+        params = {"project_id": self.repository.project_id, "entity_id": entity_id}
+        if isinstance(self.repository, SQLiteSearchRepository):
+            await self.repository.execute_query(
+                text(
+                    "DELETE FROM search_vector_embeddings WHERE rowid IN ("
+                    "SELECT id FROM search_vector_chunks "
+                    "WHERE project_id = :project_id AND entity_id = :entity_id)"
+                ),
+                params,
+            )
+
+        await self.repository.execute_query(
+            text(
+                "DELETE FROM search_vector_chunks "
+                "WHERE project_id = :project_id AND entity_id = :entity_id"
+            ),
+            params,
+        )
+
     async def index_entity_file(
         self,
         entity: Entity,
diff --git a/tests/services/test_semantic_search.py b/tests/services/test_semantic_search.py
@@ -1,7 +1,13 @@
 """Semantic search service regression tests for local SQLite search."""
 
+from datetime import datetime
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
 import pytest
 
+from basic_memory.repository import EntityRepository
+from basic_memory.repository.search_repository_base import VectorSyncBatchResult
 from basic_memory.repository.semantic_errors import (
     SemanticDependenciesMissingError,
     SemanticSearchDisabledError,
@@ -89,3 +95,125 @@ async def test_semantic_fts_mode_still_returns_observations(search_service, test
 
     assert results
     assert any(result.type == SearchItemType.OBSERVATION.value for result in results)
+
+
+@pytest.mark.asyncio
+async def test_semantic_vector_sync_skips_embed_opt_out_and_clears_vectors(
+    search_service, monkeypatch
+):
+    """Embed opt-out should clear stale vectors instead of regenerating them."""
+    repository = _sqlite_repo(search_service)
+    repository._semantic_enabled = True
+
+    monkeypatch.setattr(
+        search_service.entity_repository,
+        "find_by_id",
+        AsyncMock(return_value=SimpleNamespace(id=42, entity_metadata={"embed": False})),
+    )
+    sync_vectors = AsyncMock()
+    execute_query = AsyncMock()
+    monkeypatch.setattr(repository, "sync_entity_vectors", sync_vectors)
+    monkeypatch.setattr(repository, "execute_query", execute_query)
+
+    await search_service.sync_entity_vectors(42)
+
+    sync_vectors.assert_not_awaited()
+    assert execute_query.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_semantic_vector_sync_resumes_when_embed_opt_out_removed(
+    search_service, monkeypatch
+):
+    """Removing the opt-out should restore normal embedding sync."""
+    repository = _sqlite_repo(search_service)
+    repository._semantic_enabled = True
+
+    monkeypatch.setattr(
+        search_service.entity_repository,
+        "find_by_id",
+        AsyncMock(return_value=SimpleNamespace(id=42, entity_metadata={})),
+    )
+    sync_vectors = AsyncMock()
+    execute_query = AsyncMock()
+    monkeypatch.setattr(repository, "sync_entity_vectors", sync_vectors)
+    monkeypatch.setattr(repository, "execute_query", execute_query)
+
+    await search_service.sync_entity_vectors(42)
+
+    sync_vectors.assert_awaited_once_with(42)
+    execute_query.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_semantic_vector_sync_batch_skips_embed_opt_out_and_reports_skips(
+    search_service, monkeypatch
+):
+    """Batch vector sync should only embed eligible notes and report skipped opt-outs."""
+    repository = _sqlite_repo(search_service)
+    repository._semantic_enabled = True
+
+    monkeypatch.setattr(
+        search_service.entity_repository,
+        "find_by_ids",
+        AsyncMock(
+            return_value=[
+                SimpleNamespace(id=41, entity_metadata={"embed": False}),
+                SimpleNamespace(id=42, entity_metadata={}),
+            ]
+        ),
+    )
+    sync_batch = AsyncMock(
+        return_value=VectorSyncBatchResult(
+            entities_total=1,
+            entities_synced=1,
+            entities_failed=0,
+        )
+    )
+    execute_query = AsyncMock()
+    monkeypatch.setattr(repository, "sync_entity_vectors_batch", sync_batch)
+    monkeypatch.setattr(repository, "execute_query", execute_query)
+
+    result = await search_service.sync_entity_vectors_batch([41, 42])
+
+    sync_batch.assert_awaited_once()
+    assert sync_batch.await_args.args[0] == [42]
+    assert result.entities_total == 2
+    assert result.entities_synced == 1
+    assert result.entities_skipped == 1
+    assert execute_query.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_embed_opt_out_note_still_participates_in_fts(
+    search_service, session_maker, test_project
+):
+    """Per-note semantic opt-out should not remove the note from FTS search."""
+    entity_repo = EntityRepository(session_maker, project_id=test_project.id)
+    entity = await entity_repo.create(
+        {
+            "title": "FTS Opt Out",
+            "note_type": "note",
+            "entity_metadata": {"embed": False},
+            "content_type": "text/markdown",
+            "file_path": "test/fts-opt-out.md",
+            "permalink": "test/fts-opt-out",
+            "project_id": test_project.id,
+            "created_at": datetime.now(),
+            "updated_at": datetime.now(),
+        }
+    )
+
+    await search_service.index_entity(
+        entity,
+        content="This note should stay searchable through full text indexing.",
+    )
+
+    results = await search_service.search(
+        SearchQuery(
+            text="stay searchable",
+            retrieval_mode=SearchRetrievalMode.FTS,
+        )
+    )
+
+    assert any(result.entity_id == entity.id for result in results)
