Merge pull request #2 from CMLPlatform/feat/runnable-stack-demo-ci

feat: Make the stack runnable, demoable, and tested

Author: simonvanlierde

.env.example

@@ -6,5 +6,6 @@ GRAFANA_ADMIN_PASSWORD=change-me
 #   GRAFANA_ROOT_URL=https://grafana.example.com
 GRAFANA_ROOT_URL=http://localhost:3000
 
+# Only needed for `just up-tunnel` (production exposure via Cloudflare Tunnel).
 # From: Cloudflare Zero Trust → Networks → Tunnels → <your tunnel> → Configure → token
 CLOUDFLARE_TUNNEL_TOKEN=

.github/dependabot.yml

@@ -1,14 +1,19 @@
 version: 2
 updates:
   - package-ecosystem: "docker-compose"
-    directory: "/" 
+    directory: "/"
     schedule:
       interval: "weekly"
     groups:
       monitoring-stack-updates:
         patterns:
           - "*"
 
+  - package-ecosystem: "docker"
+    directory: "/demo"
+    schedule:
+      interval: "monthly"
+
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/workflows/ci.yml

@@ -0,0 +1,26 @@
+# Thin wrapper around the `just` contract: everything CI runs, you can run
+# locally with the same command.
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: extractions/setup-just@v4
+      - run: cp .env.example .env
+      - run: just check
+
+  smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: extractions/setup-just@v4
+      - run: cp .env.example .env
+      - run: just smoke
+      - run: just down

CHANGELOG.md

@@ -0,0 +1,42 @@
+# Changelog
+
+Notable changes to this stack. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/);
+versions follow [SemVer](https://semver.org/).
+
+## [0.1.0] - 2026-07-03
+
+First tagged release: the stack is runnable, demoable, and validated in CI.
+
+### Added
+
+- `just demo`: one-command demo — an auto-instrumented FastAPI service under
+  constant load populates Grafana with correlated traces, metrics, and logs.
+- `just check`: validation gate (compose syntax, Prometheus config + alert
+  rules, collector config, YAML, workflows, dashboard JSON), all in pinned
+  containers. Run by GitHub Actions on every push/PR, plus a `just smoke`
+  boot test.
+- Provisioned **Service Health (RED)** dashboard driven by Tempo span
+  metrics — works for any service that sends traces.
+- Prometheus alert rules: target down, OTel export failures, high span
+  error rate.
+- README: architecture diagram, demo quickstart, screenshot, design
+  decisions.
+- ADR 0001 recording the single-host OTLP-native architecture, and a
+  commented `compose.storage-s3.yml` stub documenting the storage
+  scale-out path.
+
+### Changed
+
+- `cloudflared` moved to an optional `compose.tunnel.yml` overlay and pinned
+  (was `:latest`); the core stack no longer needs a tunnel token to run.
+
+### Fixed
+
+- OTLP metrics ingestion: Prometheus 3.x renamed the OTLP receiver flag,
+  so the collector's metrics export 404ed and OTLP metrics were dropped.
+
+## [0.0.1] - 2026-06-30
+
+Initial stack: OTel Collector → Loki / Tempo / Prometheus → Grafana wired
+for cross-signal correlation, exposed via Cloudflare Tunnel, with RELab
+dashboards.

README.md

@@ -1,18 +1,58 @@
 # Observability stack
 
-Central Grafana + Loki + Tempo + Prometheus + OTel Collector, wired together
-with logs ↔ traces ↔ metrics correlation. Meant to run on one host and
-receive telemetry from multiple projects over OTLP.
+One host running Grafana, Loki, Tempo, Prometheus, and an OpenTelemetry
+Collector, wired so logs, traces, and metrics cross-link. Point any number of
+projects at it over OTLP and their telemetry lands in one place.
+
+```mermaid
+flowchart LR
+    apps["Project apps<br/>(OTLP gRPC 4317 / HTTP 4318)"] --> otel["OTel Collector<br/>(ingestion gateway)"]
+    otel -- logs --> loki[Loki]
+    otel -- traces --> tempo[Tempo]
+    otel -- metrics --> prom[Prometheus]
+    tempo -- "span metrics (RED)" --> prom
+    loki --> grafana[Grafana]
+    tempo --> grafana
+    prom --> grafana
+    grafana --- cf["Cloudflare Tunnel<br/>(optional overlay)"]
+```
 
-## Layout
+## Demo: see it work in one command
 
+```sh
+cp .env.example .env
+just demo
 ```
-compose.yml             # all services
+
+This starts the full stack plus a small auto-instrumented FastAPI service
+under constant load (`compose.demo.yml`). Give it a minute, then open
+Grafana at <http://localhost:3000> (admin / change-me):
+
+- **Dashboards → Service Health (RED)** — request rate, error rate, and
+  latency for the demo service. Latency dots are exemplars: click one to
+  open that exact trace in Tempo.
+- **Explore → Loki** — `{service_name="demo-api"}`; expand an error line
+  and follow its `trace_id` to the trace.
+- **Alerting → Alert rules** — stack-health and error-rate rules,
+  evaluated by Prometheus.
+
+![Service Health (RED) dashboard](docs/img/service-health.png)
+
+Tear it down with `just demo-down`.
+
+## Layout
+
+```sh
+compose.yml             # core services
+compose.tunnel.yml      # production overlay: Cloudflare Tunnel
+compose.demo.yml        # demo overlay: sample telemetry source
+demo/                   # the demo FastAPI service
 config/
   otel-collector.yaml   # ingestion gateway (OTLP in → Loki/Tempo/Prometheus out)
   loki.yaml             # logs
   tempo.yaml            # traces
   prometheus.yaml       # metrics
+  alerts/               # Prometheus alert rules
   grafana/              # provisioned datasources + dashboard loader
 dashboards/             # drop JSON dashboards here; Grafana auto-loads them
 ```
@@ -21,11 +61,18 @@ dashboards/             # drop JSON dashboards here; Grafana auto-loads them
 
 ```sh
 cp .env.example .env    # set GRAFANA_ADMIN_PASSWORD
-just up
+just up                 # core stack, local only
+just up-tunnel          # core stack + Cloudflare Tunnel (needs CLOUDFLARE_TUNNEL_TOKEN)
 ```
 
 Grafana: <http://localhost:3000> (admin / whatever you set).
 
+`just check` validates everything (compose files, Prometheus config and
+alert rules, collector config, YAML, workflows, dashboard JSON) in pinned
+containers — no host installs. CI runs the same command on every push and
+PR, plus a smoke test that boots the stack and waits for Grafana to report
+healthy.
+
 ## Sending telemetry from a project
 
 ### Traces + metrics (OTLP)
@@ -64,8 +111,35 @@ Logs carry labels for `service`, `env`, and `host` so you can filter in
 Grafana. Keep label cardinality low — don't add `user_id`, `request_id`,
 etc. as labels; use LogQL filters for those.
 
+## Alerting
+
+Prometheus evaluates the rules in `config/alerts/` (target down, OTel
+export failures, >5% span error rate) and Grafana surfaces them under
+Alerting → Alert rules. There is deliberately no Alertmanager: on a
+single-host stack, another service buys routing complexity before anyone
+needs it. Add one (or a Grafana contact point) when alerts must page
+someone.
+
 ## Storage
 
 Everything persists to local Docker volumes (`loki_data`, `tempo_data`,
 `prometheus_data`, `grafana_data`). Swap Loki / Tempo storage to S3-compatible
-(Backblaze B2, Cloudflare R2, Hetzner, MinIO) when you outgrow local disk.
+(Backblaze B2, Cloudflare R2, Hetzner, MinIO) when you outgrow local disk —
+`compose.storage-s3.yml` documents the concrete shape of that change, and
+`docs/adr/0001-observability-stack.md` records the architecture rationale.
+
+## Design decisions
+
+Everything enters through one gateway — the collector — so a project
+configures a single OTLP endpoint, and a backend can be swapped without
+touching any app. It runs on one host on purpose: CML's telemetry is a handful
+of services at single-digit requests per second, so distributed ingest would
+add operational weight for no gain, while one compose file stays auditable by a
+single person. When local disk runs short, S3-backed Loki and Tempo (above)
+are the documented way out.
+
+Logs carry only low-cardinality labels — `service`, `env`, `host` — and
+everything else is a query-time filter; high-cardinality labels are the usual
+way a Loki install falls over. And because Tempo's metrics generator derives
+RED metrics from spans, any service that sends traces gets the Service Health
+dashboard whether or not it emits metrics of its own.

compose.demo.yml

@@ -0,0 +1,51 @@
+# Demo overlay: a tiny auto-instrumented FastAPI service plus a load
+# generator, so the stack has real traces, metrics, and correlated logs
+# to show. Not for production.
+#
+#   just demo        # core stack + this overlay
+#   just demo-down
+
+services:
+  demo-api:
+    build: ./demo
+    restart: unless-stopped
+    depends_on: [ otel-collector ]
+    environment:
+      OTEL_SERVICE_NAME: demo-api
+      OTEL_RESOURCE_ATTRIBUTES: env=demo
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+      OTEL_TRACES_EXPORTER: otlp
+      OTEL_METRICS_EXPORTER: otlp
+      OTEL_LOGS_EXPORTER: otlp
+      # Ship Python log records via OTLP with trace context attached.
+      OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED: "true"
+      OTEL_METRIC_EXPORT_INTERVAL: "10000"
+      # Emit the *stable* HTTP semconv (http.server.request.duration in seconds,
+      # http.route) rather than the legacy names. Stable metric names are frozen,
+      # so the Service Health dashboard's app-SDK panel survives library bumps.
+      OTEL_SEMCONV_STABILITY_OPT_IN: http
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+
+  demo-load:
+    image: curlimages/curl:8.21.0
+    restart: unless-stopped
+    depends_on: [ demo-api ]
+    command:
+      - sh
+      - -c
+      - |
+        while true; do
+          curl -s -o /dev/null demo-api:8000/
+          curl -s -o /dev/null demo-api:8000/work
+          sleep 1
+        done
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"

compose.storage-s3.yml

@@ -0,0 +1,54 @@
+# STUB — the storage scale-out path (not wired up; see ADR 0001).
+#
+# When local volumes stop fitting, Loki and Tempo move their object storage
+# to any S3-compatible backend (Cloudflare R2, Backblaze B2, Hetzner, MinIO)
+# without touching the collector, Prometheus, or any client project.
+#
+# This is deliberately a commented stub: it documents the concrete shape of
+# the change so the README's scale-out claim is real, but it is not meant to
+# be started until credentials and a bucket exist. To activate:
+#
+#   1. Create s3 variants of the configs, e.g. config/loki.s3.yaml replacing
+#      `common.storage.filesystem` with:
+#
+#        common:
+#          storage:
+#            s3:
+#              endpoint: ${S3_ENDPOINT}        # e.g. <account>.r2.cloudflarestorage.com
+#              bucketnames: cml-loki
+#              access_key_id: ${S3_ACCESS_KEY_ID}
+#              secret_access_key: ${S3_SECRET_ACCESS_KEY}
+#              s3forcepathstyle: true
+#
+#      and config/tempo.s3.yaml replacing `storage.trace.backend: local` with:
+#
+#        storage:
+#          trace:
+#            backend: s3
+#            s3:
+#              endpoint: ${S3_ENDPOINT}
+#              bucket: cml-tempo
+#              access_key: ${S3_ACCESS_KEY_ID}
+#              secret_key: ${S3_SECRET_ACCESS_KEY}
+#
+#   2. Uncomment the overlay below and run:
+#        docker compose -f compose.yml -f compose.storage-s3.yml up -d
+#
+# services:
+#   loki:
+#     volumes:
+#       - ./config/loki.s3.yaml:/etc/loki/loki.yaml:ro
+#       - loki_data:/loki
+#     environment:
+#       S3_ENDPOINT: ${S3_ENDPOINT:?set S3_ENDPOINT in .env}
+#       S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
+#       S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
+#
+#   tempo:
+#     volumes:
+#       - ./config/tempo.s3.yaml:/etc/tempo/tempo.yaml:ro
+#       - tempo_data:/var/tempo
+#     environment:
+#       S3_ENDPOINT: ${S3_ENDPOINT:?set S3_ENDPOINT in .env}
+#       S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
+#       S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}

compose.tunnel.yml

@@ -0,0 +1,21 @@
+# Production overlay: expose Grafana (and the OTLP endpoints) via Cloudflare
+# Tunnel. Requires CLOUDFLARE_TUNNEL_TOKEN in .env.
+#
+#   just up-tunnel
+#
+# Token-based tunnel: routes are configured in the Cloudflare Zero Trust
+# dashboard. Create a tunnel at dash.cloudflare.com → Zero Trust → Networks →
+# Tunnels, then set the public hostnames there (e.g. grafana.example.com →
+# http://grafana:3000, otlp.example.com → http://otel-collector:4318).
+
+services:
+  cloudflared:
+    image: cloudflare/cloudflared:2026.6.1
+    restart: unless-stopped
+    depends_on: [ grafana, otel-collector ]
+    command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN:?set CLOUDFLARE_TUNNEL_TOKEN in .env}
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"

compose.yml

@@ -55,13 +55,16 @@ services:
     restart: unless-stopped
     volumes:
       - ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
+      - ./config/alerts:/etc/prometheus/alerts:ro
       - prometheus_data:/prometheus
     command:
       - --config.file=/etc/prometheus/prometheus.yaml
       - --storage.tsdb.path=/prometheus
       - --storage.tsdb.retention.time=30d
       - --web.enable-remote-write-receiver
-      - --enable-feature=native-histograms,otlp-write-receiver,exemplar-storage
+      # Prometheus 3.x: OTLP ingestion is its own flag, no longer a feature flag.
+      - --web.enable-otlp-receiver
+      - --enable-feature=native-histograms,exemplar-storage
     logging: *default-logging
 
   grafana:
@@ -82,18 +85,6 @@ services:
       GF_FEATURE_TOGGLES_ENABLE: traceqlEditor,metricsSummary,traceToProfiles
     logging: *default-logging
 
-  cloudflared:
-    image: cloudflare/cloudflared:latest
-    restart: unless-stopped
-    depends_on: [ grafana, otel-collector ]
-    # Token-based tunnel: routes are configured in the Cloudflare Zero Trust dashboard.
-    # Create a tunnel at dash.cloudflare.com → Zero Trust → Networks → Tunnels,
-    # then set the public hostnames there (e.g. grafana.example.com → http://grafana:3000,
-    # otlp.example.com → http://otel-collector:4318).
-    command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN:?set CLOUDFLARE_TUNNEL_TOKEN in .env}
-    logging: *default-logging
-    pull_policy: always
-
 volumes:
   loki_data:
   tempo_data: