chore: Cloud compatibility fixes and performance improvements (#454)

Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: phernandez

README.md

@@ -433,6 +433,39 @@ See the [Documentation](https://memory.basicmachines.co/) for more info, includi
 - [Managing multiple Projects](https://docs.basicmemory.com/guides/cli-reference/#project)
 - [Importing data from OpenAI/Claude Projects](https://docs.basicmemory.com/guides/cli-reference/#import)
 
+## Logging
+
+Basic Memory uses [Loguru](https://github.com/Delgan/loguru) for logging. The logging behavior varies by entry point:
+
+| Entry Point | Default Behavior | Use Case |
+|-------------|------------------|----------|
+| CLI commands | File only | Prevents log output from interfering with command output |
+| MCP server | File only | Stdout would corrupt the JSON-RPC protocol |
+| API server | File (local) or stdout (cloud) | Docker/cloud deployments use stdout |
+
+**Log file location:** `~/.basic-memory/basic-memory.log` (10MB rotation, 10 days retention)
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `BASIC_MEMORY_LOG_LEVEL` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR |
+| `BASIC_MEMORY_CLOUD_MODE` | `false` | When `true`, API logs to stdout with structured context |
+| `BASIC_MEMORY_ENV` | `dev` | Set to `test` for test mode (stderr only) |
+
+### Examples
+
+```bash
+# Enable debug logging
+BASIC_MEMORY_LOG_LEVEL=DEBUG basic-memory sync
+
+# View logs
+tail -f ~/.basic-memory/basic-memory.log
+
+# Cloud/Docker mode (stdout logging with structured context)
+BASIC_MEMORY_CLOUD_MODE=true uvicorn basic_memory.api.app:app
+```
+
 ## Development
 
 ### Running Tests

src/basic_memory/api/app.py

@@ -30,14 +30,17 @@
     prompt_router as v2_prompt,
     importer_router as v2_importer,
 )
-from basic_memory.config import ConfigManager
+from basic_memory.config import ConfigManager, init_api_logging
 from basic_memory.services.initialization import initialize_file_sync, initialize_app
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):  # pragma: no cover
     """Lifecycle manager for the FastAPI app. Not called in stdio mcp mode"""
 
+    # Initialize logging for API (stdout in cloud mode, file otherwise)
+    init_api_logging()
+
     app_config = ConfigManager().config
     logger.info("Starting Basic Memory API")
 

src/basic_memory/api/routers/resource_router.py

@@ -2,9 +2,9 @@
 
 import tempfile
 from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Union
 
-from fastapi import APIRouter, HTTPException, BackgroundTasks, Body
+from fastapi import APIRouter, HTTPException, BackgroundTasks, Body, Response
 from fastapi.responses import FileResponse, JSONResponse
 from loguru import logger
 
@@ -50,7 +50,7 @@ def get_entity_ids(item: SearchIndexRow) -> set[int]:
             raise ValueError(f"Unexpected type: {item.type}")
 
 
-@router.get("/{identifier:path}")
+@router.get("/{identifier:path}", response_model=None)
 async def get_resource_content(
     config: ProjectConfigDep,
     link_resolver: LinkResolverDep,
@@ -61,7 +61,7 @@ async def get_resource_content(
     identifier: str,
     page: int = 1,
     page_size: int = 10,
-) -> FileResponse:
+) -> Union[Response, FileResponse]:
     """Get resource content by identifier: name or permalink."""
     logger.debug(f"Getting content for: {identifier}")
 
@@ -92,13 +92,16 @@ async def get_resource_content(
     # return single response
     if len(results) == 1:
         entity = results[0]
-        file_path = Path(f"{config.home}/{entity.file_path}")
-        if not file_path.exists():
+        # Check file exists via file_service (for cloud compatibility)
+        if not await file_service.exists(entity.file_path):
             raise HTTPException(
                 status_code=404,
-                detail=f"File not found: {file_path}",
+                detail=f"File not found: {entity.file_path}",
             )
-        return FileResponse(path=file_path)
+        # Read content via file_service as bytes (works with both local and S3)
+        content = await file_service.read_file_bytes(entity.file_path)
+        content_type = file_service.content_type(entity.file_path)
+        return Response(content=content, media_type=content_type)
 
     # for multiple files, initialize a temporary file for writing the results
     with tempfile.NamedTemporaryFile(delete=False, mode="w", suffix=".md") as tmp_file:
@@ -192,7 +195,7 @@ async def write_resource(
         checksum = await file_service.write_file(full_path, content_str)
 
         # Get file info
-        file_stats = file_service.file_stats(full_path)
+        file_metadata = await file_service.get_file_metadata(full_path)
 
         # Determine file details
         file_name = Path(file_path).name
@@ -213,7 +216,7 @@ async def write_resource(
                     "content_type": content_type,
                     "file_path": file_path,
                     "checksum": checksum,
-                    "updated_at": datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
+                    "updated_at": file_metadata.modified_at,
                 },
             )
             status_code = 200
@@ -225,8 +228,8 @@ async def write_resource(
                 content_type=content_type,
                 file_path=file_path,
                 checksum=checksum,
-                created_at=datetime.fromtimestamp(file_stats.st_ctime).astimezone(),
-                updated_at=datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
+                created_at=file_metadata.created_at,
+                updated_at=file_metadata.modified_at,
             )
             entity = await entity_repository.add(entity)
             status_code = 201
@@ -240,9 +243,9 @@ async def write_resource(
             content={
                 "file_path": file_path,
                 "checksum": checksum,
-                "size": file_stats.st_size,
-                "created_at": file_stats.st_ctime,
-                "modified_at": file_stats.st_mtime,
+                "size": file_metadata.size,
+                "created_at": file_metadata.created_at.timestamp(),
+                "modified_at": file_metadata.modified_at.timestamp(),
             },
         )
     except Exception as e:  # pragma: no cover

src/basic_memory/api/routers/utils.py

@@ -24,8 +24,24 @@ async def to_graph_context(
     page: Optional[int] = None,
     page_size: Optional[int] = None,
 ):
+    # First pass: collect all entity IDs needed for relations
+    entity_ids_needed: set[int] = set()
+    for context_item in context_result.results:
+        for item in [context_item.primary_result] + context_item.observations + context_item.related_results:
+            if item.type == SearchItemType.RELATION:
+                if item.from_id:  # pyright: ignore
+                    entity_ids_needed.add(item.from_id)  # pyright: ignore
+                if item.to_id:
+                    entity_ids_needed.add(item.to_id)
+
+    # Batch fetch all entities at once
+    entity_lookup: dict[int, str] = {}
+    if entity_ids_needed:
+        entities = await entity_repository.find_by_ids(list(entity_ids_needed))
+        entity_lookup = {e.id: e.title for e in entities}
+
     # Helper function to convert items to summaries
-    async def to_summary(item: SearchIndexRow | ContextResultRow):
+    def to_summary(item: SearchIndexRow | ContextResultRow):
         match item.type:
             case SearchItemType.ENTITY:
                 return EntitySummary(
@@ -48,18 +64,18 @@ async def to_summary(item: SearchIndexRow | ContextResultRow):
                     created_at=item.created_at,
                 )
             case SearchItemType.RELATION:
-                from_entity = await entity_repository.find_by_id(item.from_id)  # pyright: ignore
-                to_entity = await entity_repository.find_by_id(item.to_id) if item.to_id else None
+                from_title = entity_lookup.get(item.from_id) if item.from_id else None  # pyright: ignore
+                to_title = entity_lookup.get(item.to_id) if item.to_id else None
                 return RelationSummary(
                     relation_id=item.id,
                     entity_id=item.entity_id,  # pyright: ignore
                     title=item.title,  # pyright: ignore
                     file_path=item.file_path,
                     permalink=item.permalink,  # pyright: ignore
                     relation_type=item.relation_type,  # pyright: ignore
-                    from_entity=from_entity.title if from_entity else None,
+                    from_entity=from_title,
                     from_entity_id=item.from_id,  # pyright: ignore
-                    to_entity=to_entity.title if to_entity else None,
+                    to_entity=to_title,
                     to_entity_id=item.to_id,
                     created_at=item.created_at,
                 )
@@ -70,23 +86,19 @@ async def to_summary(item: SearchIndexRow | ContextResultRow):
     hierarchical_results = []
     for context_item in context_result.results:
         # Process primary result
-        primary_result = await to_summary(context_item.primary_result)
+        primary_result = to_summary(context_item.primary_result)
 
-        # Process observations
-        observations = []
-        for obs in context_item.observations:
-            observations.append(await to_summary(obs))
+        # Process observations (always ObservationSummary, validated by context_service)
+        observations = [to_summary(obs) for obs in context_item.observations]
 
         # Process related results
-        related = []
-        for rel in context_item.related_results:
-            related.append(await to_summary(rel))
+        related = [to_summary(rel) for rel in context_item.related_results]
 
         # Add to hierarchical results
         hierarchical_results.append(
             ContextResult(
                 primary_result=primary_result,
-                observations=observations,
+                observations=observations,  # pyright: ignore[reportArgumentType]
                 related_results=related,
             )
         )

src/basic_memory/api/v2/routers/resource_router.py

@@ -11,8 +11,7 @@
 
 from pathlib import Path
 
-from fastapi import APIRouter, HTTPException
-from fastapi.responses import FileResponse
+from fastapi import APIRouter, HTTPException, Response
 from loguru import logger
 
 from basic_memory.deps import (
@@ -30,7 +29,6 @@
     ResourceResponse,
 )
 from basic_memory.utils import validate_project_path
-from datetime import datetime
 
 router = APIRouter(prefix="/resource", tags=["resources-v2"])
 
@@ -42,7 +40,7 @@ async def get_resource_content(
     config: ProjectConfigV2Dep,
     entity_service: EntityServiceV2Dep,
     file_service: FileServiceV2Dep,
-) -> FileResponse:
+) -> Response:
     """Get raw resource content by entity ID.
 
     Args:
@@ -53,7 +51,7 @@ async def get_resource_content(
         file_service: File service for reading file content
 
     Returns:
-        FileResponse with entity content
+        Response with entity content
 
     Raises:
         HTTPException: 404 if entity or file not found
@@ -76,14 +74,18 @@ async def get_resource_content(
             detail="Entity contains invalid file path",
         )
 
-    file_path = Path(f"{config.home}/{entity.file_path}")
-    if not file_path.exists():
+    # Check file exists via file_service (for cloud compatibility)
+    if not await file_service.exists(entity.file_path):
         raise HTTPException(
             status_code=404,
-            detail=f"File not found: {file_path}",
+            detail=f"File not found: {entity.file_path}",
         )
 
-    return FileResponse(path=file_path)
+    # Read content via file_service as bytes (works with both local and S3)
+    content = await file_service.read_file_bytes(entity.file_path)
+    content_type = file_service.content_type(entity.file_path)
+
+    return Response(content=content, media_type=content_type)
 
 
 @router.post("", response_model=ResourceResponse)
@@ -143,7 +145,7 @@ async def create_resource(
         checksum = await file_service.write_file(full_path, data.content)
 
         # Get file info
-        file_stats = file_service.file_stats(full_path)
+        file_metadata = await file_service.get_file_metadata(full_path)
 
         # Determine file details
         file_name = Path(data.file_path).name
@@ -157,8 +159,8 @@ async def create_resource(
             content_type=content_type,
             file_path=data.file_path,
             checksum=checksum,
-            created_at=datetime.fromtimestamp(file_stats.st_ctime).astimezone(),
-            updated_at=datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
+            created_at=file_metadata.created_at,
+            updated_at=file_metadata.modified_at,
         )
         entity = await entity_repository.add(entity)
 
@@ -170,9 +172,9 @@ async def create_resource(
             entity_id=entity.id,
             file_path=data.file_path,
             checksum=checksum,
-            size=file_stats.st_size,
-            created_at=file_stats.st_ctime,
-            modified_at=file_stats.st_mtime,
+            size=file_metadata.size,
+            created_at=file_metadata.created_at.timestamp(),
+            modified_at=file_metadata.modified_at.timestamp(),
         )
     except HTTPException:
         # Re-raise HTTP exceptions without wrapping
@@ -233,17 +235,16 @@ async def update_resource(
             )
 
         # Get full paths
-        old_full_path = Path(f"{config.home}/{entity.file_path}")
         new_full_path = Path(f"{config.home}/{target_file_path}")
 
         # If moving file, handle the move
         if data.file_path and data.file_path != entity.file_path:
             # Ensure new parent directory exists
             new_full_path.parent.mkdir(parents=True, exist_ok=True)
 
-            # If old file exists, remove it
-            if old_full_path.exists():
-                old_full_path.unlink()
+            # If old file exists, remove it via file_service (for cloud compatibility)
+            if await file_service.exists(entity.file_path):
+                await file_service.delete_file(entity.file_path)
         else:
             # Ensure directory exists for in-place update
             new_full_path.parent.mkdir(parents=True, exist_ok=True)
@@ -252,7 +253,7 @@ async def update_resource(
         checksum = await file_service.write_file(new_full_path, data.content)
 
         # Get file info
-        file_stats = file_service.file_stats(new_full_path)
+        file_metadata = await file_service.get_file_metadata(new_full_path)
 
         # Determine file details
         file_name = Path(target_file_path).name
@@ -268,7 +269,7 @@ async def update_resource(
                 "content_type": content_type,
                 "file_path": target_file_path,
                 "checksum": checksum,
-                "updated_at": datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
+                "updated_at": file_metadata.modified_at,
             },
         )
 
@@ -280,9 +281,9 @@ async def update_resource(
             entity_id=entity_id,
             file_path=target_file_path,
             checksum=checksum,
-            size=file_stats.st_size,
-            created_at=file_stats.st_ctime,
-            modified_at=file_stats.st_mtime,
+            size=file_metadata.size,
+            created_at=file_metadata.created_at.timestamp(),
+            modified_at=file_metadata.modified_at.timestamp(),
         )
     except HTTPException:
         # Re-raise HTTP exceptions without wrapping

src/basic_memory/cli/app.py

@@ -2,7 +2,7 @@
 
 import typer
 
-from basic_memory.config import ConfigManager
+from basic_memory.config import ConfigManager, init_cli_logging
 
 
 def version_callback(value: bool) -> None:
@@ -31,6 +31,9 @@ def app_callback(
 ) -> None:
     """Basic Memory - Local-first personal knowledge management."""
 
+    # Initialize logging for CLI (file only, no stdout)
+    init_cli_logging()
+
     # Run initialization for every command unless --version was specified
     if not version and ctx.invoked_subcommand is not None:
         from basic_memory.services.initialization import ensure_initialization