Merge branch 'main' into fix/integration-test-failures

Signed-off-by: Joe P <joe@basicmemory.com>

Author: jope-bm

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "python-dotenv>=1.1.0",
     "pytest-aio>=1.9.0",
     "aiofiles>=24.1.0", # Async file I/O
-    "logfire>=0.73.0", # Optional observability (disabled by default via config)
+    "logfire[fastapi]>=0.73.0", # Optional observability (disabled by default via config)
     "asyncpg>=0.30.0",
     "nest-asyncio>=1.6.0", # For Alembic migrations with Postgres
 ]

src/basic_memory/alembic/versions/f8a9b2c3d4e5_add_pg_trgm_for_fuzzy_link_resolution.py

@@ -0,0 +1,166 @@
+"""Add project_id to relation/observation and pg_trgm for fuzzy link resolution
+
+Revision ID: f8a9b2c3d4e5
+Revises: 314f1ea54dc4
+Create Date: 2025-12-01 12:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "f8a9b2c3d4e5"
+down_revision: Union[str, None] = "314f1ea54dc4"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add project_id to relation and observation tables, plus pg_trgm indexes.
+
+    This migration:
+    1. Adds project_id column to relation and observation tables (denormalization)
+    2. Backfills project_id from the associated entity
+    3. Enables pg_trgm extension for trigram-based fuzzy matching (Postgres only)
+    4. Creates GIN indexes on entity title and permalink for fast similarity searches
+    5. Creates partial index on unresolved relations for efficient bulk resolution
+    """
+    connection = op.get_bind()
+    dialect = connection.dialect.name
+
+    # -------------------------------------------------------------------------
+    # Add project_id to relation table
+    # -------------------------------------------------------------------------
+
+    # Step 1: Add project_id column as nullable first
+    op.add_column("relation", sa.Column("project_id", sa.Integer(), nullable=True))
+
+    # Step 2: Backfill project_id from entity.project_id via from_id
+    if dialect == "postgresql":
+        op.execute("""
+            UPDATE relation
+            SET project_id = entity.project_id
+            FROM entity
+            WHERE relation.from_id = entity.id
+        """)
+    else:
+        # SQLite syntax
+        op.execute("""
+            UPDATE relation
+            SET project_id = (
+                SELECT entity.project_id
+                FROM entity
+                WHERE entity.id = relation.from_id
+            )
+        """)
+
+    # Step 3: Make project_id NOT NULL and add foreign key
+    op.alter_column("relation", "project_id", nullable=False)
+    op.create_foreign_key(
+        "fk_relation_project_id",
+        "relation",
+        "project",
+        ["project_id"],
+        ["id"],
+    )
+
+    # Step 4: Create index on relation.project_id
+    op.create_index("ix_relation_project_id", "relation", ["project_id"])
+
+    # -------------------------------------------------------------------------
+    # Add project_id to observation table
+    # -------------------------------------------------------------------------
+
+    # Step 1: Add project_id column as nullable first
+    op.add_column("observation", sa.Column("project_id", sa.Integer(), nullable=True))
+
+    # Step 2: Backfill project_id from entity.project_id via entity_id
+    if dialect == "postgresql":
+        op.execute("""
+            UPDATE observation
+            SET project_id = entity.project_id
+            FROM entity
+            WHERE observation.entity_id = entity.id
+        """)
+    else:
+        # SQLite syntax
+        op.execute("""
+            UPDATE observation
+            SET project_id = (
+                SELECT entity.project_id
+                FROM entity
+                WHERE entity.id = observation.entity_id
+            )
+        """)
+
+    # Step 3: Make project_id NOT NULL and add foreign key
+    op.alter_column("observation", "project_id", nullable=False)
+    op.create_foreign_key(
+        "fk_observation_project_id",
+        "observation",
+        "project",
+        ["project_id"],
+        ["id"],
+    )
+
+    # Step 4: Create index on observation.project_id
+    op.create_index("ix_observation_project_id", "observation", ["project_id"])
+
+    # Postgres-specific: pg_trgm and GIN indexes
+    if dialect == "postgresql":
+        # Enable pg_trgm extension for fuzzy string matching
+        op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+
+        # Create trigram indexes on entity table for fuzzy matching
+        # GIN indexes with gin_trgm_ops support similarity searches
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_entity_title_trgm
+            ON entity USING gin (title gin_trgm_ops)
+        """)
+
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_entity_permalink_trgm
+            ON entity USING gin (permalink gin_trgm_ops)
+        """)
+
+        # Create partial index on unresolved relations for efficient bulk resolution
+        # This makes "WHERE to_id IS NULL AND project_id = X" queries very fast
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_relation_unresolved
+            ON relation (project_id, to_name)
+            WHERE to_id IS NULL
+        """)
+
+        # Create index on relation.to_name for join performance in bulk resolution
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_relation_to_name
+            ON relation (to_name)
+        """)
+
+
+def downgrade() -> None:
+    """Remove project_id from relation/observation and pg_trgm indexes."""
+    connection = op.get_bind()
+    dialect = connection.dialect.name
+
+    if dialect == "postgresql":
+        # Drop Postgres-specific indexes
+        op.execute("DROP INDEX IF EXISTS idx_relation_to_name")
+        op.execute("DROP INDEX IF EXISTS idx_relation_unresolved")
+        op.execute("DROP INDEX IF EXISTS idx_entity_permalink_trgm")
+        op.execute("DROP INDEX IF EXISTS idx_entity_title_trgm")
+        # Note: We don't drop the pg_trgm extension as other code may depend on it
+
+    # Drop project_id from observation
+    op.drop_index("ix_observation_project_id", table_name="observation")
+    op.drop_constraint("fk_observation_project_id", "observation", type_="foreignkey")
+    op.drop_column("observation", "project_id")
+
+    # Drop project_id from relation
+    op.drop_index("ix_relation_project_id", table_name="relation")
+    op.drop_constraint("fk_relation_project_id", "relation", type_="foreignkey")
+    op.drop_column("relation", "project_id")

src/basic_memory/api/app.py

@@ -100,8 +100,6 @@ async def lifespan(app: FastAPI):  # pragma: no cover
 app.include_router(project.project_resource_router)
 app.include_router(management.router)
 
-# Auth routes are handled by FastMCP automatically when auth is enabled
-
 
 @app.exception_handler(Exception)
 async def exception_handler(request, exc):  # pragma: no cover

src/basic_memory/api/routers/resource_router.py

@@ -25,6 +25,17 @@
 router = APIRouter(prefix="/resource", tags=["resources"])
 
 
+def _mtime_to_datetime(entity: EntityModel) -> datetime:
+    """Convert entity mtime (file modification time) to datetime.
+
+    Returns the file's actual modification time, falling back to updated_at
+    if mtime is not available.
+    """
+    if entity.mtime:
+        return datetime.fromtimestamp(entity.mtime).astimezone()
+    return entity.updated_at
+
+
 def get_entity_ids(item: SearchIndexRow) -> set[int]:
     match item.type:
         case SearchItemType.ENTITY:
@@ -97,7 +108,7 @@ async def get_resource_content(
             # Read content for each entity
             content = await file_service.read_entity_content(result)
             memory_url = normalize_memory_url(result.permalink)
-            modified_date = result.updated_at.isoformat()
+            modified_date = _mtime_to_datetime(result).isoformat()
             checksum = result.checksum[:8] if result.checksum else ""
 
             # Prepare the delimited content

src/basic_memory/config.py

@@ -100,6 +100,23 @@ class BasicMemoryConfig(BaseSettings):
         description="Database connection URL. For Postgres, use postgresql+asyncpg://user:pass@host:port/db. If not set, SQLite will use default path.",
     )
 
+    # Database connection pool configuration (Postgres only)
+    db_pool_size: int = Field(
+        default=20,
+        description="Number of connections to keep in the pool (Postgres only)",
+        gt=0,
+    )
+    db_pool_overflow: int = Field(
+        default=40,
+        description="Max additional connections beyond pool_size under load (Postgres only)",
+        gt=0,
+    )
+    db_pool_recycle: int = Field(
+        default=180,
+        description="Recycle connections after N seconds to prevent stale connections. Default 180s works well with Neon's ~5 minute scale-to-zero (Postgres only)",
+        gt=0,
+    )
+
     # Watch service configuration
     sync_delay: int = Field(
         default=1000, description="Milliseconds to wait after changes before syncing", gt=0

src/basic_memory/db.py

@@ -190,21 +190,37 @@ def enable_wal_mode(dbapi_conn, connection_record):
     return engine
 
 
-def _create_postgres_engine(db_url: str) -> AsyncEngine:
+def _create_postgres_engine(db_url: str, config: BasicMemoryConfig) -> AsyncEngine:
     """Create Postgres async engine with appropriate configuration.
 
     Args:
         db_url: Postgres connection URL (postgresql+asyncpg://...)
+        config: BasicMemoryConfig with pool settings
 
     Returns:
         Configured async engine for Postgres
     """
-    # Postgres with asyncpg - use standard async connection
+    # Use NullPool connection issues.
+    # Assume connection pooler like PgBouncer handles connection pooling.
     engine = create_async_engine(
         db_url,
         echo=False,
-        pool_pre_ping=True,  # Verify connections before using them
+        poolclass=NullPool,  # No pooling - fresh connection per request
+        connect_args={
+            # Disable statement cache to avoid issues with prepared statements on reconnect
+            "statement_cache_size": 0,
+            # Allow 30s for commands (Neon cold start can take 2-5s, sometimes longer)
+            "command_timeout": 30,
+            # Allow 30s for initial connection (Neon wake-up time)
+            "timeout": 30,
+            "server_settings": {
+                "application_name": "basic-memory",
+                # Statement timeout for queries (30s to allow for cold start)
+                "statement_timeout": "30s",
+            },
+        },
     )
+    logger.debug("Created Postgres engine with NullPool (no connection pooling)")
 
     return engine
 
@@ -228,7 +244,7 @@ def _create_engine_and_session(
     # Delegate to backend-specific engine creation
     # Check explicit POSTGRES type first, then config setting
     if db_type == DatabaseType.POSTGRES or config.database_backend == DatabaseBackend.POSTGRES:
-        engine = _create_postgres_engine(db_url)
+        engine = _create_postgres_engine(db_url, config)
     else:
         engine = _create_sqlite_engine(db_url, db_type)
 

src/basic_memory/deps.py

@@ -368,11 +368,10 @@ async def get_markdown_processor_v2(entity_parser: EntityParserV2Dep) -> Markdow
 async def get_file_service(
     project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
 ) -> FileService:
+    file_service = FileService(project_config.home, markdown_processor)
     logger.debug(
-        f"Creating FileService for project: {project_config.name}, base_path: {project_config.home}"
+        f"Created FileService for project: {project_config.name}, base_path: {project_config.home} "
     )
-    file_service = FileService(project_config.home, markdown_processor)
-    logger.debug(f"Created FileService for project: {file_service} ")
     return file_service
 
 
@@ -382,11 +381,10 @@ async def get_file_service(
 async def get_file_service_v2(
     project_config: ProjectConfigV2Dep, markdown_processor: MarkdownProcessorV2Dep
 ) -> FileService:
+    file_service = FileService(project_config.home, markdown_processor)
     logger.debug(
-        f"Creating FileService for project: {project_config.name}, base_path: {project_config.home}"
+        f"Created FileService for project: {project_config.name}, base_path: {project_config.home}"
     )
-    file_service = FileService(project_config.home, markdown_processor)
-    logger.debug(f"Created FileService for project: {file_service} ")
     return file_service
 
 

src/basic_memory/markdown/entity_parser.py

@@ -22,10 +22,12 @@
     Relation,
 )
 from basic_memory.utils import parse_tags
+import logfire
 
 md = MarkdownIt().use(observation_plugin).use(relation_plugin)
 
 
+@logfire.instrument()
 def normalize_frontmatter_value(value: Any) -> Any:
     """Normalize frontmatter values to safe types for processing.
 
@@ -87,6 +89,7 @@ def normalize_frontmatter_value(value: Any) -> Any:
     return value
 
 
+@logfire.instrument()
 def normalize_frontmatter_metadata(metadata: dict) -> dict:
     """Normalize all values in frontmatter metadata dict.
 
@@ -109,6 +112,7 @@ class EntityContent:
     relations: list[Relation] = field(default_factory=list)
 
 
+@logfire.instrument()
 def parse(content: str) -> EntityContent:
     """Parse markdown content into EntityMarkdown."""
 
@@ -167,6 +171,7 @@ def parse_date(self, value: Any) -> Optional[datetime]:
                 return parsed
         return None
 
+    @logfire.instrument()
     async def parse_file(self, path: Path | str) -> EntityMarkdown:
         """Parse markdown file into EntityMarkdown."""
 
@@ -188,6 +193,7 @@ def get_file_path(self, path):
         """Get absolute path for a file using the base path for the project."""
         return self.base_path / path
 
+    @logfire.instrument()
     async def parse_file_content(self, absolute_path, file_content):
         """Parse markdown content from file stats.
 
@@ -205,6 +211,7 @@ async def parse_file_content(self, absolute_path, file_content):
             ctime=file_stats.st_ctime,
         )
 
+    @logfire.instrument()
     async def parse_markdown_content(
         self,
         file_path: Path,