perf(core): reduce postgres vector sync work (#723)

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

.claude/settings.json

@@ -1,3 +1,22 @@
 {
-  "enabledPlugins": {}
+  "$schema": "https://json.schemastore.org/claude-code-settings.json",
+  "env": {
+    "CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR": "1",
+    "CLAUDE_CODE_DISABLE_FEEDBACK_SURVEY": "1",
+    "DISABLE_TELEMETRY": "1",
+    "CLAUDE_CODE_NO_FLICKER": "1",
+    "CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING": "1"
+  },
+  "permissions": {
+    "allow": [
+      "Bash(just fast-check)",
+      "Bash(just check)",
+      "Bash(just fix)",
+      "Bash(just typecheck)",
+      "Bash(just lint)",
+      "Bash(just test)"
+    ],
+    "deny": []
+  },
+  "enableAllProjectMcpServers": true
 }

src/basic_memory/alembic/versions/m6h7i8j9k0l1_add_vector_sync_fingerprints.py

@@ -0,0 +1,84 @@
+"""Persist vector sync fingerprints on chunk metadata.
+
+Revision ID: m6h7i8j9k0l1
+Revises: l5g6h7i8j9k0
+Create Date: 2026-04-07 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "m6h7i8j9k0l1"
+down_revision: Union[str, None] = "l5g6h7i8j9k0"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add entity fingerprint + embedding model metadata to Postgres chunk rows.
+
+    Trigger: vector sync now fast-skips unchanged entities using persisted
+    semantic fingerprints.
+    Why: chunk rows already own the per-entity derived metadata we diff against,
+    so persisting the fingerprint on that table avoids a second sync-state table.
+    Outcome: existing rows get empty-string placeholders and will be refreshed on
+    the next vector sync before they become eligible for skip checks.
+    """
+    connection = op.get_bind()
+    if connection.dialect.name != "postgresql":
+        return
+
+    op.execute(
+        """
+        ALTER TABLE search_vector_chunks
+        ADD COLUMN IF NOT EXISTS entity_fingerprint TEXT
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE search_vector_chunks
+        ADD COLUMN IF NOT EXISTS embedding_model TEXT
+        """
+    )
+    op.execute(
+        """
+        UPDATE search_vector_chunks
+        SET entity_fingerprint = COALESCE(entity_fingerprint, ''),
+            embedding_model = COALESCE(embedding_model, '')
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE search_vector_chunks
+        ALTER COLUMN entity_fingerprint SET NOT NULL
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE search_vector_chunks
+        ALTER COLUMN embedding_model SET NOT NULL
+        """
+    )
+
+
+def downgrade() -> None:
+    """Remove vector sync fingerprint columns from Postgres chunk rows."""
+    connection = op.get_bind()
+    if connection.dialect.name != "postgresql":
+        return
+
+    op.execute(
+        """
+        ALTER TABLE search_vector_chunks
+        DROP COLUMN IF EXISTS embedding_model
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE search_vector_chunks
+        DROP COLUMN IF EXISTS entity_fingerprint
+        """
+    )

src/basic_memory/cli/commands/cloud/cloud_utils.py

@@ -116,12 +116,12 @@ async def sync_project(project_name: str, force_full: bool = False) -> None:
 
     Args:
         project_name: Name of project to sync
-        force_full: If True, force a full scan bypassing watermark optimization
+        force_full: ignored, kept for backwards compatibility
     """
     try:
         from basic_memory.cli.commands.command_utils import run_sync
 
-        await run_sync(project=project_name, force_full=force_full)
+        await run_sync(project=project_name)
     except Exception as e:
         raise CloudUtilsError(f"Failed to sync project '{project_name}': {e}") from e
 

src/basic_memory/cli/commands/cloud/upload_command.py

@@ -142,7 +142,7 @@ async def _upload():
         if sync and not dry_run:
             console.print(f"[blue]Syncing project '{project}'...[/blue]")
             try:
-                await sync_project(project, force_full=True)
+                await sync_project(project)
             except Exception as e:
                 console.print(f"[yellow]Warning: Sync failed: {e}[/yellow]")
                 console.print("[dim]Files uploaded but may not be indexed yet[/dim]")

src/basic_memory/cli/commands/doctor.py

@@ -101,7 +101,7 @@ async def run_doctor() -> None:
                 console.print("[green]OK[/green] Manual file written")
 
                 sync_data = await project_client.sync(
-                    project_id, force_full=True, run_in_background=False
+                    project_id, force_full=False, run_in_background=False
                 )
                 sync_report = SyncReportResponse.model_validate(sync_data)
                 if sync_report.total == 0:

src/basic_memory/config.py

@@ -198,6 +198,12 @@ class BasicMemoryConfig(BaseSettings):
         description="Batch size for vector sync orchestration flushes.",
         gt=0,
     )
+    semantic_postgres_prepare_concurrency: int = Field(
+        default=4,
+        description="Number of Postgres entity prepare tasks to run concurrently during vector sync. Postgres only; keep this low to avoid overdriving the database connection pool.",
+        gt=0,
+        le=16,
+    )
     semantic_embedding_cache_dir: str | None = Field(
         default=None,
         description="Optional cache directory for FastEmbed model artifacts.",

src/basic_memory/models/search.py

@@ -104,6 +104,8 @@
     chunk_key TEXT NOT NULL,
     chunk_text TEXT NOT NULL,
     source_hash TEXT NOT NULL,
+    entity_fingerprint TEXT NOT NULL,
+    embedding_model TEXT NOT NULL,
     updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
     UNIQUE (project_id, entity_id, chunk_key)
 )
@@ -124,6 +126,8 @@
     chunk_key TEXT NOT NULL,
     chunk_text TEXT NOT NULL,
     source_hash TEXT NOT NULL,
+    entity_fingerprint TEXT NOT NULL,
+    embedding_model TEXT NOT NULL,
     updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
 )
 """)