diff --git a/.env.example b/.env.example index 33d7e4f..de19807 100644 --- a/.env.example +++ b/.env.example @@ -72,9 +72,12 @@ BROKER_URL=localhost:9092 # Dual-model usage: gpt-4o-mini for classify (~500ms, cheap), gpt-4o for deep_analyze (~1-2s). OPENAI_API_KEY=sk-your-real-key-here -# OpenTelemetry (distributed tracing → Jaeger) -# Set OTEL_ENABLED=true when running with docker compose (Jaeger is included) -OTEL_ENABLED=false +# OpenTelemetry (distributed tracing → Grafana Tempo) +# Tracing is on by default in docker compose; the Tempo collector ships with the +# monitoring profile (`just up-monitoring`). Export failures are non-fatal when +# monitoring is down. For host-run dev (`uv run`), Tempo's OTLP port is exposed +# on 127.0.0.1:4317, so the localhost endpoint below works as-is. +OTEL_ENABLED=true OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 OTEL_SERVICE_NAME=api-obs-ingestor diff --git a/Dockerfile b/Dockerfile index 5f59e63..2eae966 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,8 +21,10 @@ ENV UV_COMPILE_BYTECODE=1 \ # Install deps first (better layer caching) # --extra ai: the LangGraph incident-triage agent (Phase 3) and /analyze's RAG # path are core ingestor features, not optional demos — install their deps. +# --extra tracing: OTel SDK + OTLP exporter; without it setup_tracing() degrades +# to a no-op and spans/trace_id correlation are silently lost (post-MVP Phase 0). COPY pyproject.toml uv.lock ./ -RUN uv sync --no-dev --frozen --no-install-project --extra ai +RUN uv sync --no-dev --frozen --no-install-project --extra ai --extra tracing # Stage 2: Final image — slim, no build tools, non-root user FROM python:3.14-slim@sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061 AS runtime diff --git a/Justfile b/Justfile index 95f35ad..8023455 100644 --- a/Justfile +++ b/Justfile @@ -116,11 +116,11 @@ watch: echo "" docker compose -f docker-compose.yml -f docker-compose.dev.yml watch ingestor dashboard -# Start monitoring stack (Prometheus, Grafana, Loki, Promtail, Alertmanager, Mailpit). +# Start monitoring stack (Prometheus, Grafana, Loki, Promtail, Tempo, Alertmanager, Mailpit). up-monitoring: @just stack-info - docker compose --profile monitoring up -d prometheus grafana loki promtail alertmanager mailpit - echo "monitoring ready — Grafana http://127.0.0.1:3000, Prometheus http://127.0.0.1:9090" + docker compose --profile monitoring up -d prometheus grafana loki promtail tempo alertmanager mailpit + echo "monitoring ready — Grafana http://127.0.0.1:3000, Prometheus http://127.0.0.1:9090, Tempo http://127.0.0.1:3200" diff --git a/docker-compose.yml b/docker-compose.yml index 28b8220..0d741e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,6 +121,14 @@ services: INFERENCE_URL: http://inference:8001 LOG_FORMAT: json SERVICE_VERSION: ${SERVICE_VERSION:-0.1.0} + # Tracing on by default for the compose profile; spans export to the + # Tempo collector (monitoring profile). Export failures are non-fatal + # when monitoring is down. Override via .env. + OTEL_ENABLED: ${OTEL_ENABLED:-true} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://tempo:4317} + # Matches container_name so Tempo's trace-to-logs tag mapping + # (service.name -> Loki `container` label) resolves. + OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-api-obs-ingestor} ports: - '8000:8000' depends_on: @@ -399,6 +407,22 @@ services: networks: - api-obs + tempo: + image: grafana/tempo:2.7.2 + profiles: [monitoring] + container_name: api-obs-tempo + volumes: + - ./infra/monitoring/tempo.yml:/etc/tempo.yml:ro + ports: + # Tempo HTTP API (Grafana datasource queries; also handy for curl checks) + - '127.0.0.1:3200:3200' + # OTLP gRPC ingest — loopback-exposed so host-run services (uv run) can export too + - '127.0.0.1:4317:4317' + command: ['-config.file=/etc/tempo.yml'] + security_opt: [no-new-privileges:true] + networks: + - api-obs + alertmanager: image: prom/alertmanager:v0.28.1 profiles: [monitoring] @@ -545,7 +569,7 @@ services: # - broker (message broker) # - ingestor (FastAPI microservice) # - edge (HTTPS ingress proxy, started by default with `just up`) -# - prometheus, grafana, loki, promtail, alertmanager, mailpit (monitoring, optional via `just up-monitoring`) +# - prometheus, grafana, loki, promtail, tempo, alertmanager, mailpit (monitoring, optional via `just up-monitoring`) # - floci-az (Azure emulator, optional via --profile azure) # - floci-aws (AWS emulator, optional via --profile aws) # - floci-gcp (GCP emulator, optional via --profile gcp) diff --git a/infra/monitoring/grafana/provisioning/datasources/loki.yml b/infra/monitoring/grafana/provisioning/datasources/loki.yml index 6f0ef78..0143fa0 100644 --- a/infra/monitoring/grafana/provisioning/datasources/loki.yml +++ b/infra/monitoring/grafana/provisioning/datasources/loki.yml @@ -3,7 +3,16 @@ apiVersion: 1 datasources: - name: Loki type: loki + uid: loki access: proxy url: http://loki:3100 jsonData: maxLines: 1000 + # Logs -> traces: extract trace_id from the JSON log line and link it + # to the Tempo datasource ($$ escapes Grafana's env-var interpolation). + derivedFields: + - name: TraceID + datasourceUid: tempo + matcherRegex: '"trace_id":\s*"([a-f0-9]{32})"' + url: '$${__value.raw}' + urlDisplayLabel: 'View trace' diff --git a/infra/monitoring/grafana/provisioning/datasources/tempo.yml b/infra/monitoring/grafana/provisioning/datasources/tempo.yml new file mode 100644 index 0000000..8380e1f --- /dev/null +++ b/infra/monitoring/grafana/provisioning/datasources/tempo.yml @@ -0,0 +1,22 @@ +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + jsonData: + # Traces -> logs: from a span, jump to the Loki lines that carry the + # same trace_id (the ingestor injects it via services/ingestor/core/logging.py). + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: '-5m' + spanEndTimeShift: '5m' + # service.name (OTEL_SERVICE_NAME=api-obs-ingestor) maps onto the + # promtail-derived `container` label — the two are kept identical. + tags: + - key: 'service.name' + value: 'container' + filterByTraceID: true + filterBySpanID: false diff --git a/infra/monitoring/tempo.yml b/infra/monitoring/tempo.yml new file mode 100644 index 0000000..60ac520 --- /dev/null +++ b/infra/monitoring/tempo.yml @@ -0,0 +1,29 @@ +# Grafana Tempo — local monolithic trace collector for the compose monitoring +# profile. Receives OTLP gRPC spans from the app services and serves the +# Grafana Tempo datasource (infra/monitoring/grafana/provisioning/datasources/tempo.yml). + +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + # Since Tempo 2.3 receivers bind to localhost by default — + # 0.0.0.0 is required for other containers to reach the collector. + endpoint: 0.0.0.0:4317 + +storage: + trace: + backend: local + local: + # /tmp is writable for the image's non-root user; traces are ephemeral + # by design (consistent with the unpersisted local Prometheus/Loki). + path: /tmp/tempo/blocks + wal: + path: /tmp/tempo/wal + +compactor: + compaction: + block_retention: 48h diff --git a/libs/platform/tracing.py b/libs/platform/tracing.py index f0cb325..453310e 100644 --- a/libs/platform/tracing.py +++ b/libs/platform/tracing.py @@ -1,11 +1,11 @@ """OpenTelemetry distributed tracing setup. -Wires the TracerProvider with an OTLP gRPC exporter (Jaeger all-in-one or +Wires the TracerProvider with an OTLP gRPC exporter (Grafana Tempo or any OTLP collector) and auto-instruments FastAPI. Usage (from app/main.py lifespan): - from services.ingestor.core.tracing import setup_tracing + from libs.platform.tracing import setup_tracing setup_tracing( app, endpoint=settings.otel_exporter_otlp_endpoint, @@ -52,8 +52,8 @@ def setup_tracing( Args: app: The FastAPI application instance to instrument. - endpoint: OTLP gRPC endpoint URL (e.g., http://jaeger:4317). - service_name: Name shown in the Jaeger service dropdown. + endpoint: OTLP gRPC endpoint URL (e.g., http://tempo:4317). + service_name: Name shown in the trace backend's service dropdown. """ global _initialized if _initialized: @@ -108,7 +108,7 @@ def get_trace_id() -> str | None: - The span context is invalid (e.g., no-op tracer) Used by log formatters to inject trace_id into structured log output - so log observations can be correlated with Jaeger traces. + so log lines can be correlated with traces in Grafana Tempo. """ try: from opentelemetry import trace diff --git a/services/ingestor/config.py b/services/ingestor/config.py index bfcacee..ba9ab64 100644 --- a/services/ingestor/config.py +++ b/services/ingestor/config.py @@ -218,14 +218,14 @@ class Settings(BaseSettings): otel_exporter_otlp_endpoint: str = Field( default="http://localhost:4317", description=( - "OTLP gRPC endpoint for trace export (e.g., http://jaeger:4317). " + "OTLP gRPC endpoint for trace export (e.g., http://tempo:4317). " "Matches the standard OTel SDK env var OTEL_EXPORTER_OTLP_ENDPOINT." ), ) otel_service_name: str = Field( default="ingestor", - description="Service name shown in Jaeger / OTel collector UI.", + description="Service name shown in Grafana Tempo / OTel collector UI.", ) # ============ Sentry (error tracking) ============ diff --git a/services/ingestor/main.py b/services/ingestor/main.py index 6a7e651..ce19eb4 100644 --- a/services/ingestor/main.py +++ b/services/ingestor/main.py @@ -198,16 +198,11 @@ async def lifespan(app: FastAPI): # STARTUP # ======================================================================== - # Init distributed tracing first (trace_id available for all subsequent logs) setup_sentry() - # Init distributed tracing (trace_id available for all subsequent logs) - if settings.otel_enabled: - setup_tracing( - app, - endpoint=settings.otel_exporter_otlp_endpoint, - service_name=settings.otel_service_name, - ) + # NOTE: setup_tracing() runs at module level (after the add_middleware + # calls) — by lifespan time Starlette has already built the middleware + # stack, so FastAPIInstrumentor would never enter the request path. logger.info("startup", extra={"event": "application_started"}) _validate_production_security_settings() @@ -623,6 +618,17 @@ async def get_openapi_schema() -> dict: app.add_middleware(TenantMiddleware) app.add_middleware(CorrelationIdMiddleware) +# Init distributed tracing last so the OTel middleware wraps the whole stack: +# CorrelationIdMiddleware's request_start/request_end logs then run inside the +# server span and pick up trace_id. Must happen before the app starts serving +# (middleware cannot be added once Starlette builds its stack). +if settings.otel_enabled: + setup_tracing( + app, + endpoint=settings.otel_exporter_otlp_endpoint, + service_name=settings.otel_service_name, + ) + _ROUTER_MODULES = [ "auth", "agent", diff --git a/services/ingestor/tests/unit/core/test_tracing.py b/services/ingestor/tests/unit/core/test_tracing.py new file mode 100644 index 0000000..aa744ba --- /dev/null +++ b/services/ingestor/tests/unit/core/test_tracing.py @@ -0,0 +1,35 @@ +"""Regression tests for libs/platform/tracing.get_trace_id. + +Post-MVP Phase 0 guard: get_trace_id() must return the active span's trace ID +as a 32-char hex string so structured logs stay correlated with Tempo traces +(services/ingestor/core/logging.py injects it as `trace_id`). +""" + +from __future__ import annotations + +import pytest + +from libs.platform.tracing import get_trace_id + + +def test_get_trace_id_is_none_without_active_span() -> None: + """Outside any span (or without OTel installed) the helper degrades to None.""" + assert get_trace_id() is None + + +def test_get_trace_id_returns_32_hex_inside_active_span() -> None: + """Inside an active span the helper returns that span's 32-char hex trace ID.""" + pytest.importorskip("opentelemetry", reason="tracing extra not installed") + from opentelemetry.sdk.trace import TracerProvider + + # Local provider — avoids mutating the process-global tracer provider, + # which is one-shot and would leak into other tests. + tracer = TracerProvider().get_tracer(__name__) + with tracer.start_as_current_span("regression-guard") as span: + trace_id = get_trace_id() + + assert trace_id is not None + assert len(trace_id) == 32 + assert trace_id == format(span.get_span_context().trace_id, "032x") + + assert get_trace_id() is None