feat: add Logfire phased instrumentation (#692)

Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: phernandez

docs/logfire-instrumentation-strategy.md

@@ -205,6 +205,51 @@ doctor:
     BASIC_MEMORY_CONFIG_DIR="$TMP_CONFIG" \
     ./.venv/bin/python -m basic_memory.cli.main doctor --local
 
+# Run an isolated Logfire smoke workflow for local trace inspection
+telemetry-smoke:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    TMP_HOME=$(mktemp -d)
+    TMP_CONFIG=$(mktemp -d)
+    TMP_PROJECT=$(mktemp -d)
+    export HOME="$TMP_HOME"
+    export BASIC_MEMORY_ENV="${BASIC_MEMORY_ENV:-dev}"
+    export BASIC_MEMORY_HOME="$TMP_PROJECT/home-root"
+    export BASIC_MEMORY_CONFIG_DIR="$TMP_CONFIG"
+    export BASIC_MEMORY_NO_PROMOS=1
+    export BASIC_MEMORY_LOG_LEVEL="${BASIC_MEMORY_LOG_LEVEL:-INFO}"
+    export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED="${BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED:-false}"
+    export BASIC_MEMORY_LOGFIRE_ENABLED="${BASIC_MEMORY_LOGFIRE_ENABLED:-true}"
+    export BASIC_MEMORY_LOGFIRE_ENVIRONMENT="${BASIC_MEMORY_LOGFIRE_ENVIRONMENT:-telemetry-smoke}"
+    if [[ -z "${BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE:-}" ]]; then
+        if [[ -n "${LOGFIRE_TOKEN:-}" ]]; then
+            export BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE=true
+        else
+            export BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE=false
+        fi
+    fi
+    mkdir -p "$BASIC_MEMORY_HOME"
+    echo "Telemetry smoke setup:"
+    echo "  logfire_enabled=$BASIC_MEMORY_LOGFIRE_ENABLED"
+    echo "  send_to_logfire=$BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE"
+    echo "  log_level=$BASIC_MEMORY_LOG_LEVEL"
+    echo "  semantic_search_enabled=$BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED"
+    echo "  logfire_environment=$BASIC_MEMORY_LOGFIRE_ENVIRONMENT"
+    echo "  project_path=$TMP_PROJECT"
+    ./.venv/bin/python -m basic_memory.cli.main project add telemetry-smoke "$TMP_PROJECT" --default --local
+    ./.venv/bin/python -m basic_memory.cli.main tool write-note --title "Telemetry Smoke" --folder notes --content "hello from smoke" --project telemetry-smoke --local
+    ./.venv/bin/python -m basic_memory.cli.main tool read-note notes/telemetry-smoke --project telemetry-smoke --local
+    ./.venv/bin/python -m basic_memory.cli.main tool edit-note notes/telemetry-smoke --operation append --content $'\n\nsmoke edit line' --project telemetry-smoke --local
+    ./.venv/bin/python -m basic_memory.cli.main tool build-context notes/telemetry-smoke --project telemetry-smoke --local --page-size 5 --max-related 5
+    ./.venv/bin/python -m basic_memory.cli.main tool search-notes telemetry --project telemetry-smoke --local
+    ./.venv/bin/python -m basic_memory.cli.main doctor --local
+    echo ""
+    echo "Telemetry smoke complete."
+    echo "Search Logfire for:"
+    echo "  service_name: basic-memory-cli"
+    echo "  environment: $BASIC_MEMORY_LOGFIRE_ENVIRONMENT"
+    echo "  span names: mcp.tool.write_note, mcp.tool.read_note, mcp.tool.edit_note, mcp.tool.build_context, mcp.tool.search_notes, sync.project.run"
+
 
 # Update all dependencies to latest versions
 update-deps:

justfile

@@ -58,6 +58,9 @@ Documentation = "https://github.com/basicmachines-co/basic-memory#readme"
 basic-memory = "basic_memory.cli.main:app"
 bm = "basic_memory.cli.main:app"
 
+[project.optional-dependencies]
+telemetry = ["logfire>=4.19.0"]
+
 [build-system]
 requires = ["hatchling", "uv-dynamic-versioning>=0.7.0"]
 build-backend = "hatchling.build"
@@ -83,6 +86,7 @@ target-version = "py312"
 
 [dependency-groups]
 dev = [
+    "logfire>=4.19.0",
     "gevent>=24.11.1",
     "icecream>=2.1.3",
     "pytest>=8.3.4",

pyproject.toml

@@ -25,6 +25,7 @@
     list_projects,
     synchronize_projects,
 )
+from basic_memory import telemetry
 from basic_memory.config import init_api_logging
 from basic_memory.services.exceptions import EntityAlreadyExistsError
 from basic_memory.services.initialization import initialize_app
@@ -43,30 +44,39 @@ async def lifespan(app: FastAPI):  # pragma: no cover
     set_container(container)
     app.state.container = container
 
-    logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
+    with telemetry.operation(
+        "api.lifecycle.startup",
+        entrypoint="api",
+        mode=container.mode.name.lower(),
+    ):
+        logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
 
-    await initialize_app(container.config)
+        await initialize_app(container.config)
 
-    # Cache database connections in app state for performance
-    logger.info("Initializing database and caching connections...")
-    engine, session_maker = await container.init_database()
-    app.state.engine = engine
-    app.state.session_maker = session_maker
-    logger.info("Database connections cached in app state")
+        # Cache database connections in app state for performance
+        logger.info("Initializing database and caching connections...")
+        engine, session_maker = await container.init_database()
+        app.state.engine = engine
+        app.state.session_maker = session_maker
+        logger.info("Database connections cached in app state")
 
-    # Create and start sync coordinator (lifecycle centralized in coordinator)
-    sync_coordinator = container.create_sync_coordinator()
-    await sync_coordinator.start()
-    app.state.sync_coordinator = sync_coordinator
+        # Create and start sync coordinator (lifecycle centralized in coordinator)
+        sync_coordinator = container.create_sync_coordinator()
+        await sync_coordinator.start()
+        app.state.sync_coordinator = sync_coordinator
 
     # Proceed with startup
     yield
 
     # Shutdown - coordinator handles clean task cancellation
-    logger.info("Shutting down Basic Memory API")
-    await sync_coordinator.stop()
-
-    await container.shutdown_database()
+    with telemetry.operation(
+        "api.lifecycle.shutdown",
+        entrypoint="api",
+        mode=container.mode.name.lower(),
+    ):
+        logger.info("Shutting down Basic Memory API")
+        await sync_coordinator.stop()
+        await container.shutdown_database()
 
 
 # Initialize FastAPI app

src/basic_memory/api/app.py

@@ -6,6 +6,7 @@
 
 from fastapi import APIRouter, HTTPException, Path
 
+from basic_memory import telemetry
 from basic_memory.api.v2.utils import to_search_results
 from basic_memory.repository.semantic_errors import (
     SemanticDependenciesMissingError,
@@ -47,29 +48,39 @@ async def search(
     Returns:
         SearchResponse with paginated search results
     """
-    offset = (page - 1) * page_size
-    # Fetch one extra item to detect whether more pages exist (N+1 trick)
-    fetch_limit = page_size + 1
-    try:
-        results = await search_service.search(query, limit=fetch_limit, offset=offset)
-    except SemanticSearchDisabledError as exc:
-        raise HTTPException(status_code=400, detail=str(exc)) from exc
-    except SemanticDependenciesMissingError as exc:
-        raise HTTPException(status_code=400, detail=str(exc)) from exc
-    except ValueError as exc:
-        raise HTTPException(status_code=400, detail=str(exc)) from exc
-
-    has_more = len(results) > page_size
-    if has_more:
-        results = results[:page_size]
-
-    search_results = await to_search_results(entity_service, results)
-    return SearchResponse(
-        results=search_results,
-        current_page=page,
+    with telemetry.operation(
+        "api.request.search",
+        entrypoint="api",
+        page=page,
         page_size=page_size,
-        has_more=has_more,
-    )
+        retrieval_mode=query.retrieval_mode.value,
+        has_text_query=bool(query.text and query.text.strip()),
+        has_title_query=bool(query.title),
+        has_permalink_query=bool(query.permalink or query.permalink_match),
+    ):
+        offset = (page - 1) * page_size
+        # Fetch one extra item to detect whether more pages exist (N+1 trick)
+        fetch_limit = page_size + 1
+        try:
+            results = await search_service.search(query, limit=fetch_limit, offset=offset)
+        except SemanticSearchDisabledError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+        except SemanticDependenciesMissingError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+        has_more = len(results) > page_size
+        if has_more:
+            results = results[:page_size]
+
+        search_results = await to_search_results(entity_service, results)
+        return SearchResponse(
+            results=search_results,
+            current_page=page,
+            page_size=page_size,
+            has_more=has_more,
+        )
 
 
 @router.post("/search/reindex")

src/basic_memory/api/v2/routers/search_router.py

@@ -12,6 +12,7 @@
 from basic_memory.cli.container import CliContainer, set_container  # noqa: E402
 from basic_memory.cli.promo import maybe_show_cloud_promo, maybe_show_init_line  # noqa: E402
 from basic_memory.config import init_cli_logging  # noqa: E402
+from basic_memory import telemetry  # noqa: E402
 
 
 def version_callback(value: bool) -> None:
@@ -42,6 +43,14 @@ def app_callback(
 
     # Initialize logging for CLI (file only, no stdout)
     init_cli_logging()
+    command_name = ctx.invoked_subcommand or "root"
+    ctx.with_resource(
+        telemetry.operation(
+            f"cli.command.{command_name}",
+            entrypoint="cli",
+            command_name=command_name,
+        )
+    )
 
     # --- Composition Root ---
     # Create container and read config (single point of config access)

src/basic_memory/cli/app.py

@@ -6,14 +6,16 @@
 import shutil
 from dataclasses import dataclass
 from datetime import datetime
+from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, Literal, Optional, List, Tuple
-from enum import Enum
 
 from loguru import logger
 from pydantic import AliasChoices, BaseModel, Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+from basic_memory import __version__
+from basic_memory.telemetry import configure_telemetry
 from basic_memory.utils import setup_logging, generate_permalink
 
 
@@ -140,6 +142,24 @@ class BasicMemoryConfig(BaseSettings):
     # overridden by ~/.basic-memory/config.json
     log_level: str = "INFO"
 
+    # Optional Logfire telemetry (disabled by default)
+    logfire_enabled: bool = Field(
+        default=False,
+        description="Enable Logfire instrumentation for local development or managed deployments.",
+    )
+    logfire_send_to_logfire: bool = Field(
+        default=False,
+        description="When true, allow Logfire to export telemetry to the configured backend.",
+    )
+    logfire_service_name: str = Field(
+        default="basic-memory",
+        description="Base service name used when constructing entrypoint-specific Logfire service names.",
+    )
+    logfire_environment: str | None = Field(
+        default=None,
+        description="Optional override for Logfire environment. Defaults to env when unset.",
+    )
+
     # Database configuration
     database_backend: DatabaseBackend = Field(
         default=DatabaseBackend.SQLITE,
@@ -955,33 +975,50 @@ def save_basic_memory_config(file_path: Path, config: BasicMemoryConfig) -> None
 # Logging initialization functions for different entry points
 
 
-def init_cli_logging() -> None:  # pragma: no cover
+def _configure_logfire_for_entrypoint(entrypoint: str) -> None:
+    """Configure optional Logfire telemetry for a specific entrypoint."""
+    config = ConfigManager().config
+    service_name = f"{config.logfire_service_name}-{entrypoint}"
+    environment = config.logfire_environment or config.env
+    configure_telemetry(
+        service_name=service_name,
+        environment=environment,
+        service_version=__version__,
+        enable_logfire=config.logfire_enabled,
+        send_to_logfire=config.logfire_send_to_logfire,
+    )
+
+
+def init_cli_logging() -> None:
     """Initialize logging for CLI commands - file only.
 
     CLI commands should not log to stdout to avoid interfering with
     command output and shell integration.
     """
     log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
+    _configure_logfire_for_entrypoint("cli")
     setup_logging(log_level=log_level, log_to_file=True)
 
 
-def init_mcp_logging() -> None:  # pragma: no cover
+def init_mcp_logging() -> None:
     """Initialize logging for MCP server - file only.
 
     MCP server must not log to stdout as it would corrupt the
     JSON-RPC protocol communication.
     """
     log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
+    _configure_logfire_for_entrypoint("mcp")
     setup_logging(log_level=log_level, log_to_file=True)
 
 
-def init_api_logging() -> None:  # pragma: no cover
+def init_api_logging() -> None:
     """Initialize logging for API server.
 
     Cloud mode (BASIC_MEMORY_CLOUD_MODE=1): stdout with structured context
     Local mode: file only
     """
     log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
+    _configure_logfire_for_entrypoint("api")
     cloud_mode = os.getenv("BASIC_MEMORY_CLOUD_MODE", "").lower() in ("1", "true")
     if cloud_mode:
         setup_logging(log_level=log_level, log_to_stdout=True, structured_context=True)

src/basic_memory/config.py

@@ -5,6 +5,7 @@
 from httpx import ASGITransport, AsyncClient, Timeout
 from loguru import logger
 
+from basic_memory import telemetry
 from basic_memory.api.app import app as fastapi_app
 from basic_memory.config import ConfigManager, ProjectMode
 
@@ -43,21 +44,26 @@ def _asgi_client(timeout: Timeout) -> AsyncClient:
 
 async def _resolve_cloud_token(config) -> str:
     """Resolve cloud token with API key preferred, OAuth fallback."""
-    token = config.cloud_api_key
-    if token:
-        return token
-
-    from basic_memory.cli.auth import CLIAuth
-
-    auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
-    token = await auth.get_valid_token()
-    if token:
-        return token
-
-    raise RuntimeError(
-        "Cloud routing requested but no credentials found. "
-        "Run 'bm cloud api-key save <key>' or 'bm cloud login' first."
-    )
+    with telemetry.span(
+        "routing.resolve_cloud_credentials",
+        has_api_key=bool(config.cloud_api_key),
+    ):
+        token = config.cloud_api_key
+        if token:
+            return token
+
+        from basic_memory.cli.auth import CLIAuth
+
+        auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
+        token = await auth.get_valid_token()
+        if token:
+            return token
+
+        logger.error("Cloud routing requested but no credentials were available")
+        raise RuntimeError(
+            "Cloud routing requested but no credentials found. "
+            "Run 'bm cloud api-key save <key>' or 'bm cloud login' first."
+        )
 
 
 @asynccontextmanager