fix: semantic embeddings not generated on fresh DB or upgrade

The previous backfill trigger relied on Alembic revision tracking, but
alembic_version only stores the head revision — intermediate revisions
(like the backfill trigger) are invisible after a multi-step upgrade or
fresh DB creation.

Three changes fix this:

1. Replace Alembic revision check with a simple "entities exist but
   embeddings are empty" check that works regardless of migration path
2. Generate embeddings during sync — after FTS indexing, batch-embed all
   synced entities at the end of the sync operation
3. Add background backfill at MCP startup for the upgrade path (entities
   already exist, no embeddings) without blocking server readiness

Also adds clear startup logging for semantic embedding status so issues
are easy to spot in the logs.

📋 Covers: fresh DB, upgrade from pre-embedding version, db reset,
   interrupted backfill

Signed-off-by: Pedro Hernandez <pedro@basicmachines.co>
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/db.py

@@ -43,40 +43,37 @@
 _engine: Optional[AsyncEngine] = None
 _session_maker: Optional[async_sessionmaker[AsyncSession]] = None
 
-# Alembic revision that enables one-time automatic embedding backfill.
-SEMANTIC_EMBEDDING_BACKFILL_REVISION = "i2c3d4e5f6g7"
 
-
-async def _load_applied_alembic_revisions(
+async def _needs_semantic_embedding_backfill(
+    app_config: BasicMemoryConfig,
     session_maker: async_sessionmaker[AsyncSession],
-) -> set[str]:
-    """Load applied Alembic revisions from alembic_version.
+) -> bool:
+    """Check if entities exist but vector embeddings are empty.
 
-    Returns an empty set when the version table does not exist yet
-    (fresh database before first migration).
+    This is the reliable way to detect that embeddings need to be generated,
+    regardless of how migrations were applied (fresh DB, upgrade, reset, etc.).
     """
+    if not app_config.semantic_search_enabled:
+        return False
+
     try:
         async with scoped_session(session_maker) as session:
-            result = await session.execute(text("SELECT version_num FROM alembic_version"))
-            return {str(row[0]) for row in result.fetchall() if row[0]}
+            entity_count = (
+                await session.execute(text("SELECT COUNT(*) FROM entity"))
+            ).scalar() or 0
+            if entity_count == 0:
+                return False
+
+            # Check if vector chunks table exists and is empty
+            embedding_count = (
+                await session.execute(text("SELECT COUNT(*) FROM search_vector_chunks"))
+            ).scalar() or 0
+
+            return embedding_count == 0
     except Exception as exc:
-        error_message = str(exc).lower()
-        if "alembic_version" in error_message and (
-            "no such table" in error_message or "does not exist" in error_message
-        ):
-            return set()
-        raise
-
-
-def _should_run_semantic_embedding_backfill(
-    revisions_before_upgrade: set[str],
-    revisions_after_upgrade: set[str],
-) -> bool:
-    """Check if this migration run newly applied the backfill-trigger revision."""
-    return (
-        SEMANTIC_EMBEDDING_BACKFILL_REVISION in revisions_after_upgrade
-        and SEMANTIC_EMBEDDING_BACKFILL_REVISION not in revisions_before_upgrade
-    )
+        # Table might not exist yet (pre-migration)
+        logger.debug(f"Could not check embedding status: {exc}")
+        return False
 
 
 async def _run_semantic_embedding_backfill(
@@ -480,26 +477,9 @@ async def run_migrations(
     Note: Alembic tracks which migrations have been applied via the alembic_version table,
     so it's safe to call this multiple times - it will only run pending migrations.
     """
-    logger.debug("Running database migrations...")
+    logger.info("Running database migrations...")
     temp_engine: AsyncEngine | None = None
     try:
-        revisions_before_upgrade: set[str] = set()
-        # Trigger: run_migrations() can be invoked before module-level session maker is set.
-        # Why: we still need reliable before/after revision detection for one-time backfill.
-        # Outcome: create a short-lived session maker when needed, then dispose it immediately.
-        if _session_maker is None:
-            precheck_engine, temp_session_maker = _create_engine_and_session(
-                app_config.database_path,
-                database_type,
-                app_config,
-            )
-            try:
-                revisions_before_upgrade = await _load_applied_alembic_revisions(temp_session_maker)
-            finally:
-                await precheck_engine.dispose()
-        else:
-            revisions_before_upgrade = await _load_applied_alembic_revisions(_session_maker)
-
         # Get the absolute path to the alembic directory relative to this file
         alembic_dir = Path(__file__).parent / "alembic"
         config = Config()
@@ -519,7 +499,7 @@ async def run_migrations(
         config.set_main_option("sqlalchemy.url", db_url)
 
         command.upgrade(config, "head")
-        logger.debug("Migrations completed successfully")
+        logger.info("Migrations completed successfully")
 
         # Get session maker - ensure we don't trigger recursive migration calls
         if _session_maker is None:
@@ -541,12 +521,14 @@ async def run_migrations(
         else:
             await SQLiteSearchRepository(session_maker, 1).init_search_index()
 
-        revisions_after_upgrade = await _load_applied_alembic_revisions(session_maker)
-        if _should_run_semantic_embedding_backfill(
-            revisions_before_upgrade,
-            revisions_after_upgrade,
-        ):
-            await _run_semantic_embedding_backfill(app_config, session_maker)
+        # Check if backfill is needed — actual backfill runs in background
+        # from the MCP server lifespan to avoid blocking startup.
+        if await _needs_semantic_embedding_backfill(app_config, session_maker):
+            logger.info(
+                "Semantic embeddings missing — backfill will run in background after startup"
+            )
+        else:
+            logger.info("Semantic embeddings: up to date")
     except Exception as e:  # pragma: no cover
         logger.error(f"Error running migrations: {e}")
         raise

src/basic_memory/mcp/server.py

@@ -2,18 +2,71 @@
 Basic Memory FastMCP server.
 """
 
+import asyncio
 import time
 from contextlib import asynccontextmanager
 
 from fastmcp import FastMCP
 from loguru import logger
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import async_sessionmaker, AsyncSession
 
 from basic_memory import db
 from basic_memory.cli.auth import CLIAuth
+from basic_memory.config import BasicMemoryConfig
+from basic_memory.db import (
+    scoped_session,
+    _needs_semantic_embedding_backfill,
+    _run_semantic_embedding_backfill,
+)
 from basic_memory.mcp.container import McpContainer, set_container
 from basic_memory.services.initialization import initialize_app
 
 
+async def _log_embedding_status(session_maker: async_sessionmaker[AsyncSession]) -> None:
+    """Log a clear summary of semantic embedding status at startup."""
+    try:
+        async with scoped_session(session_maker) as session:
+            entity_count = (
+                await session.execute(text("SELECT COUNT(*) FROM entity"))
+            ).scalar() or 0
+            chunk_count = (
+                await session.execute(text("SELECT COUNT(*) FROM search_vector_chunks"))
+            ).scalar() or 0
+            embedding_count = (
+                await session.execute(text("SELECT COUNT(*) FROM search_vector_embeddings_rowids"))
+            ).scalar() or 0
+
+        if entity_count == 0:
+            logger.info("Semantic embeddings: no entities yet")
+        elif embedding_count == 0:
+            logger.warning(
+                f"Semantic embeddings: EMPTY — {entity_count} entities have no embeddings. "
+                "Backfill running in background..."
+            )
+        else:
+            logger.info(
+                f"Semantic embeddings: {embedding_count} embeddings "
+                f"across {chunk_count} chunks for {entity_count} entities"
+            )
+    except Exception as exc:
+        logger.debug(f"Could not check embedding status at startup: {exc}")
+
+
+async def _background_embedding_backfill(
+    config: BasicMemoryConfig,
+    session_maker: async_sessionmaker[AsyncSession],
+) -> None:
+    """Run semantic embedding backfill in the background without blocking startup."""
+    try:
+        if await _needs_semantic_embedding_backfill(config, session_maker):
+            logger.info("Background embedding backfill starting...")
+            await _run_semantic_embedding_backfill(config, session_maker)
+            await _log_embedding_status(session_maker)
+    except Exception as exc:
+        logger.error(f"Background embedding backfill failed: {exc}")
+
+
 @asynccontextmanager
 async def lifespan(app: FastMCP):
     """Lifecycle manager for the MCP server.
@@ -70,6 +123,16 @@ async def lifespan(app: FastMCP):
     # Initialize app (runs migrations, reconciles projects)
     await initialize_app(container.config)
 
+    # Log embedding status so it's easy to spot in the logs
+    backfill_task: asyncio.Task | None = None  # type: ignore[type-arg]
+    if config.semantic_search_enabled and db._session_maker is not None:
+        await _log_embedding_status(db._session_maker)
+        # Launch backfill in background so MCP server is ready immediately
+        backfill_task = asyncio.create_task(
+            _background_embedding_backfill(config, db._session_maker),
+            name="embedding-backfill",
+        )
+
     # Create and start sync coordinator (lifecycle centralized in coordinator)
     sync_coordinator = container.create_sync_coordinator()
     await sync_coordinator.start()
@@ -79,6 +142,15 @@ async def lifespan(app: FastMCP):
     finally:
         # Shutdown - coordinator handles clean task cancellation
         logger.debug("Shutting down Basic Memory MCP server")
+
+        # Cancel embedding backfill if still running
+        if backfill_task is not None and not backfill_task.done():
+            backfill_task.cancel()
+            try:
+                await backfill_task
+            except asyncio.CancelledError:
+                logger.info("Background embedding backfill cancelled during shutdown")
+
         await sync_coordinator.stop()
 
         # Only shutdown DB if we created it (not if test fixture provided it)

src/basic_memory/sync/sync_service.py

@@ -293,12 +293,16 @@ async def sync(
         for path in report.deleted:
             await self.handle_delete(path)
 
-        # then new and modified
+        # then new and modified — collect entity IDs for batch vector embedding
+        synced_entity_ids: list[int] = []
+
         for path in report.new:
             entity, _ = await self.sync_file(path, new=True)
 
+            if entity is not None:
+                synced_entity_ids.append(entity.id)
             # Track if file was skipped
-            if entity is None and await self._should_skip_file(path):
+            elif await self._should_skip_file(path):
                 failure_info = self._file_failures[path]
                 report.skipped_files.append(
                     SkippedFile(
@@ -312,8 +316,10 @@ async def sync(
         for path in report.modified:
             entity, _ = await self.sync_file(path, new=False)
 
+            if entity is not None:
+                synced_entity_ids.append(entity.id)
             # Track if file was skipped
-            if entity is None and await self._should_skip_file(path):
+            elif await self._should_skip_file(path):
                 failure_info = self._file_failures[path]
                 report.skipped_files.append(
                     SkippedFile(
@@ -331,6 +337,26 @@ async def sync(
         else:
             logger.info("Skipping relation resolution - no file changes detected")
 
+        # Batch-generate vector embeddings for all synced entities
+        if synced_entity_ids and self.app_config.semantic_search_enabled:
+            try:
+                logger.info(
+                    f"Generating semantic embeddings for {len(synced_entity_ids)} entities..."
+                )
+                batch_result = await self.search_service.sync_entity_vectors_batch(
+                    synced_entity_ids
+                )
+                logger.info(
+                    f"Semantic embeddings complete: "
+                    f"synced={batch_result.entities_synced}, "
+                    f"failed={batch_result.entities_failed}"
+                )
+            except SemanticDependenciesMissingError:
+                logger.warning(
+                    "Semantic search dependencies missing — vector embeddings skipped. "
+                    "Run 'bm reindex --embeddings' after resolving the dependency issue."
+                )
+
         # Update scan watermark after successful sync
         # Use the timestamp from sync start (not end) to ensure we catch files
         # created during the sync on the next iteration

tests/mcp/test_project_context.py

@@ -31,6 +31,7 @@ async def test_returns_none_when_no_default_and_no_project(config_manager, monke
     config_manager.save_config(cfg)
 
     monkeypatch.delenv("BASIC_MEMORY_MCP_PROJECT", raising=False)
+
     # Prevent API fallback from returning a project via stale dependency overrides
     async def _no_api_fallback():
         return None
@@ -117,6 +118,7 @@ async def test_returns_none_when_no_default(config_manager, monkeypatch):
     config_manager.save_config(cfg)
 
     monkeypatch.delenv("BASIC_MEMORY_MCP_PROJECT", raising=False)
+
     # Prevent API fallback from returning a project via stale dependency overrides
     async def _no_api_fallback():
         return None