feat(observability): wire OTel tracing end-to-end with Tempo (post-MVP Phase 0)

.env.example

@@ -72,9 +72,12 @@ BROKER_URL=localhost:9092
 # Dual-model usage: gpt-4o-mini for classify (~500ms, cheap), gpt-4o for deep_analyze (~1-2s).
 OPENAI_API_KEY=sk-your-real-key-here
 
-# OpenTelemetry (distributed tracing → Jaeger)
-# Set OTEL_ENABLED=true when running with docker compose (Jaeger is included)
-OTEL_ENABLED=false
+# OpenTelemetry (distributed tracing → Grafana Tempo)
+# Tracing is on by default in docker compose; the Tempo collector ships with the
+# monitoring profile (`just up-monitoring`). Export failures are non-fatal when
+# monitoring is down. For host-run dev (`uv run`), Tempo's OTLP port is exposed
+# on 127.0.0.1:4317, so the localhost endpoint below works as-is.
+OTEL_ENABLED=true
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 OTEL_SERVICE_NAME=api-obs-ingestor
 

Dockerfile

@@ -21,8 +21,10 @@ ENV UV_COMPILE_BYTECODE=1 \
 # Install deps first (better layer caching)
 # --extra ai: the LangGraph incident-triage agent (Phase 3) and /analyze's RAG
 # path are core ingestor features, not optional demos — install their deps.
+# --extra tracing: OTel SDK + OTLP exporter; without it setup_tracing() degrades
+# to a no-op and spans/trace_id correlation are silently lost (post-MVP Phase 0).
 COPY pyproject.toml uv.lock ./
-RUN uv sync --no-dev --frozen --no-install-project --extra ai
+RUN uv sync --no-dev --frozen --no-install-project --extra ai --extra tracing
 
 # Stage 2: Final image — slim, no build tools, non-root user
 FROM python:3.14-slim@sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061 AS runtime

Justfile

@@ -116,11 +116,11 @@ watch:
     echo ""
     docker compose -f docker-compose.yml -f docker-compose.dev.yml watch ingestor dashboard
 
-# Start monitoring stack (Prometheus, Grafana, Loki, Promtail, Alertmanager, Mailpit).
+# Start monitoring stack (Prometheus, Grafana, Loki, Promtail, Tempo, Alertmanager, Mailpit).
 up-monitoring:
     @just stack-info
-    docker compose --profile monitoring up -d prometheus grafana loki promtail alertmanager mailpit
-    echo "monitoring ready — Grafana http://127.0.0.1:3000, Prometheus http://127.0.0.1:9090"
+    docker compose --profile monitoring up -d prometheus grafana loki promtail tempo alertmanager mailpit
+    echo "monitoring ready — Grafana http://127.0.0.1:3000, Prometheus http://127.0.0.1:9090, Tempo http://127.0.0.1:3200"
 
 
 

docker-compose.yml

@@ -121,6 +121,14 @@ services:
       INFERENCE_URL: http://inference:8001
       LOG_FORMAT: json
       SERVICE_VERSION: ${SERVICE_VERSION:-0.1.0}
+      # Tracing on by default for the compose profile; spans export to the
+      # Tempo collector (monitoring profile). Export failures are non-fatal
+      # when monitoring is down. Override via .env.
+      OTEL_ENABLED: ${OTEL_ENABLED:-true}
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://tempo:4317}
+      # Matches container_name so Tempo's trace-to-logs tag mapping
+      # (service.name -> Loki `container` label) resolves.
+      OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-api-obs-ingestor}
     ports:
       - '8000:8000'
     depends_on:
@@ -399,6 +407,22 @@ services:
     networks:
       - api-obs
 
+  tempo:
+    image: grafana/tempo:2.7.2
+    profiles: [monitoring]
+    container_name: api-obs-tempo
+    volumes:
+      - ./infra/monitoring/tempo.yml:/etc/tempo.yml:ro
+    ports:
+      # Tempo HTTP API (Grafana datasource queries; also handy for curl checks)
+      - '127.0.0.1:3200:3200'
+      # OTLP gRPC ingest — loopback-exposed so host-run services (uv run) can export too
+      - '127.0.0.1:4317:4317'
+    command: ['-config.file=/etc/tempo.yml']
+    security_opt: [no-new-privileges:true]
+    networks:
+      - api-obs
+
   alertmanager:
     image: prom/alertmanager:v0.28.1
     profiles: [monitoring]
@@ -545,7 +569,7 @@ services:
 #   - broker (message broker)
 #   - ingestor (FastAPI microservice)
 #   - edge (HTTPS ingress proxy, started by default with `just up`)
-#   - prometheus, grafana, loki, promtail, alertmanager, mailpit (monitoring, optional via `just up-monitoring`)
+#   - prometheus, grafana, loki, promtail, tempo, alertmanager, mailpit (monitoring, optional via `just up-monitoring`)
 #   - floci-az (Azure emulator, optional via --profile azure)
 #   - floci-aws (AWS emulator, optional via --profile aws)
 #   - floci-gcp (GCP emulator, optional via --profile gcp)

infra/monitoring/grafana/provisioning/datasources/loki.yml

@@ -3,7 +3,16 @@ apiVersion: 1
 datasources:
   - name: Loki
     type: loki
+    uid: loki
     access: proxy
     url: http://loki:3100
     jsonData:
       maxLines: 1000
+      # Logs -> traces: extract trace_id from the JSON log line and link it
+      # to the Tempo datasource ($$ escapes Grafana's env-var interpolation).
+      derivedFields:
+        - name: TraceID
+          datasourceUid: tempo
+          matcherRegex: '"trace_id":\s*"([a-f0-9]{32})"'
+          url: '$${__value.raw}'
+          urlDisplayLabel: 'View trace'

infra/monitoring/grafana/provisioning/datasources/tempo.yml

@@ -0,0 +1,22 @@
+apiVersion: 1
+
+datasources:
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    url: http://tempo:3200
+    jsonData:
+      # Traces -> logs: from a span, jump to the Loki lines that carry the
+      # same trace_id (the ingestor injects it via services/ingestor/core/logging.py).
+      tracesToLogsV2:
+        datasourceUid: loki
+        spanStartTimeShift: '-5m'
+        spanEndTimeShift: '5m'
+        # service.name (OTEL_SERVICE_NAME=api-obs-ingestor) maps onto the
+        # promtail-derived `container` label — the two are kept identical.
+        tags:
+          - key: 'service.name'
+            value: 'container'
+        filterByTraceID: true
+        filterBySpanID: false

infra/monitoring/tempo.yml

@@ -0,0 +1,29 @@
+# Grafana Tempo — local monolithic trace collector for the compose monitoring
+# profile. Receives OTLP gRPC spans from the app services and serves the
+# Grafana Tempo datasource (infra/monitoring/grafana/provisioning/datasources/tempo.yml).
+
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          # Since Tempo 2.3 receivers bind to localhost by default —
+          # 0.0.0.0 is required for other containers to reach the collector.
+          endpoint: 0.0.0.0:4317
+
+storage:
+  trace:
+    backend: local
+    local:
+      # /tmp is writable for the image's non-root user; traces are ephemeral
+      # by design (consistent with the unpersisted local Prometheus/Loki).
+      path: /tmp/tempo/blocks
+    wal:
+      path: /tmp/tempo/wal
+
+compactor:
+  compaction:
+    block_retention: 48h

libs/platform/tracing.py

@@ -1,11 +1,11 @@
 """OpenTelemetry distributed tracing setup.
 
-Wires the TracerProvider with an OTLP gRPC exporter (Jaeger all-in-one or
+Wires the TracerProvider with an OTLP gRPC exporter (Grafana Tempo or
 any OTLP collector) and auto-instruments FastAPI.
 
 Usage (from app/main.py lifespan):
 
-    from services.ingestor.core.tracing import setup_tracing
+    from libs.platform.tracing import setup_tracing
     setup_tracing(
         app,
         endpoint=settings.otel_exporter_otlp_endpoint,
@@ -52,8 +52,8 @@ def setup_tracing(
 
     Args:
         app: The FastAPI application instance to instrument.
-        endpoint: OTLP gRPC endpoint URL (e.g., http://jaeger:4317).
-        service_name: Name shown in the Jaeger service dropdown.
+        endpoint: OTLP gRPC endpoint URL (e.g., http://tempo:4317).
+        service_name: Name shown in the trace backend's service dropdown.
     """
     global _initialized
     if _initialized:
@@ -108,7 +108,7 @@ def get_trace_id() -> str | None:
     - The span context is invalid (e.g., no-op tracer)
 
     Used by log formatters to inject trace_id into structured log output
-    so log observations can be correlated with Jaeger traces.
+    so log lines can be correlated with traces in Grafana Tempo.
     """
     try:
         from opentelemetry import trace

services/ingestor/config.py

@@ -218,14 +218,14 @@ class Settings(BaseSettings):
     otel_exporter_otlp_endpoint: str = Field(
         default="http://localhost:4317",
         description=(
-            "OTLP gRPC endpoint for trace export (e.g., http://jaeger:4317). "
+            "OTLP gRPC endpoint for trace export (e.g., http://tempo:4317). "
             "Matches the standard OTel SDK env var OTEL_EXPORTER_OTLP_ENDPOINT."
         ),
     )
 
     otel_service_name: str = Field(
         default="ingestor",
-        description="Service name shown in Jaeger / OTel collector UI.",
+        description="Service name shown in Grafana Tempo / OTel collector UI.",
     )
 
     # ============ Sentry (error tracking) ============

services/ingestor/main.py

@@ -198,16 +198,11 @@ async def lifespan(app: FastAPI):
     # STARTUP
     # ========================================================================
 
-    # Init distributed tracing first (trace_id available for all subsequent logs)
     setup_sentry()
 
-    # Init distributed tracing (trace_id available for all subsequent logs)
-    if settings.otel_enabled:
-        setup_tracing(
-            app,
-            endpoint=settings.otel_exporter_otlp_endpoint,
-            service_name=settings.otel_service_name,
-        )
+    # NOTE: setup_tracing() runs at module level (after the add_middleware
+    # calls) — by lifespan time Starlette has already built the middleware
+    # stack, so FastAPIInstrumentor would never enter the request path.
 
     logger.info("startup", extra={"event": "application_started"})
     _validate_production_security_settings()
@@ -623,6 +618,17 @@ async def get_openapi_schema() -> dict:
 app.add_middleware(TenantMiddleware)
 app.add_middleware(CorrelationIdMiddleware)
 
+# Init distributed tracing last so the OTel middleware wraps the whole stack:
+# CorrelationIdMiddleware's request_start/request_end logs then run inside the
+# server span and pick up trace_id. Must happen before the app starts serving
+# (middleware cannot be added once Starlette builds its stack).
+if settings.otel_enabled:
+    setup_tracing(
+        app,
+        endpoint=settings.otel_exporter_otlp_endpoint,
+        service_name=settings.otel_service_name,
+    )
+
 _ROUTER_MODULES = [
     "auth",
     "agent",

services/ingestor/tests/unit/core/test_tracing.py

@@ -0,0 +1,35 @@
+"""Regression tests for libs/platform/tracing.get_trace_id.
+
+Post-MVP Phase 0 guard: get_trace_id() must return the active span's trace ID
+as a 32-char hex string so structured logs stay correlated with Tempo traces
+(services/ingestor/core/logging.py injects it as `trace_id`).
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from libs.platform.tracing import get_trace_id
+
+
+def test_get_trace_id_is_none_without_active_span() -> None:
+    """Outside any span (or without OTel installed) the helper degrades to None."""
+    assert get_trace_id() is None
+
+
+def test_get_trace_id_returns_32_hex_inside_active_span() -> None:
+    """Inside an active span the helper returns that span's 32-char hex trace ID."""
+    pytest.importorskip("opentelemetry", reason="tracing extra not installed")
+    from opentelemetry.sdk.trace import TracerProvider
+
+    # Local provider — avoids mutating the process-global tracer provider,
+    # which is one-shot and would leak into other tests.
+    tracer = TracerProvider().get_tracer(__name__)
+    with tracer.start_as_current_span("regression-guard") as span:
+        trace_id = get_trace_id()
+
+        assert trace_id is not None
+        assert len(trace_id) == 32
+        assert trace_id == format(span.get_span_context().trace_id, "032x")
+
+    assert get_trace_id() is None