diff --git a/.env.example b/.env.example index 7e9e6d9..a581ecf 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,17 @@ GRAFANA_ADMIN_PASSWORD=change-me # GRAFANA_ROOT_URL=https://grafana.example.com GRAFANA_ROOT_URL=http://localhost:3000 +# Bearer token every telemetry sender must present (Authorization: Bearer ). +# The default only suits local use — generate a real one for production, e.g.: openssl rand -hex 32 +OTLP_AUTH_TOKEN=local-dev-token + +# Where Alertmanager delivers alert notifications (any webhook: ntfy, Slack, …). +# Leave empty to run without delivery; failures are logged and harmless. +ALERT_WEBHOOK_URL= +# Dead man's switch ping target (e.g. https://hc-ping.com/). The Watchdog +# alert posts here every 5m; alert externally when pings stop. +HEARTBEAT_URL= + # Only needed for `just up-tunnel` (production exposure via Cloudflare Tunnel). # From: Cloudflare Zero Trust → Networks → Tunnels → → Configure → token CLOUDFLARE_TUNNEL_TOKEN= diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a0ce903..4149d6c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,12 +8,22 @@ updates: monitoring-stack-updates: patterns: - "*" + ignore: + # Tempo 3.x is a re-architecture requiring a config migration; upgrade + # deliberately, not via a routine bump. See comment in compose.yml. + - dependency-name: "grafana/tempo" + update-types: ["version-update:semver-major"] - package-ecosystem: "docker" directory: "/demo" schedule: interval: "monthly" + - package-ecosystem: "pip" + directory: "/demo" + schedule: + interval: "monthly" + - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 024130c..982f69c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,9 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: check: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index fb29868..d29ad9d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ .env *.local.yml *.local.yaml +backups/ +infra/.terraform/ +infra/terraform.tfstate* +infra/*.tfvars diff --git a/CHANGELOG.md b/CHANGELOG.md index 79bc055..d255bbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,38 @@ Notable changes to this stack. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow [SemVer](https://semver.org/). +## [Unreleased] + +Reliability hardening: the stack becomes operable, not just runnable. + +### Added + +- Bearer-token auth on OTLP ingestion (`OTLP_AUTH_TOKEN`); senders use + `OTEL_EXPORTER_OTLP_HEADERS`. +- Host self-monitoring: node-exporter, `HostDiskSpaceLow` alert (the + storage backstop — Loki/Tempo have no total-size cap), **Stack Health** + dashboard. +- Alert delivery: Alertmanager routes to `ALERT_WEBHOOK_URL`; an + always-firing `Watchdog` heartbeats to `HEARTBEAT_URL` (dead man's + switch). +- `just backup` / `just restore` — crash-consistent volume snapshots, + exercised end-to-end. +- `docs/RUNBOOK.md` and `docs/ONBOARDING.md` with verified telemetry + templates (zero-code Python, plain OTLP, Loki Docker driver, Grafana + Alloy). +- `infra/`: OpenTofu for the Cloudflare tunnel, ingress routes, and DNS. + +### Changed + +- Prometheus storage now bounded by size as well as time + (`--storage.tsdb.retention.size=15GB`). + +### Fixed + +- Tempo held at 2.x: a routine bump to 3.0 crash-looped on the 2.x config + (3.0 is a re-architecture). Dependabot now skips Tempo majors so the + migration happens deliberately. + ## [0.1.0] - 2026-07-03 First tagged release: the stack is runnable, demoable, and validated in CI. diff --git a/README.md b/README.md index 810ea20..a32f8c6 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,37 @@ One host running Grafana, Loki, Tempo, Prometheus, and an OpenTelemetry Collector, wired so logs, traces, and metrics cross-link. Point any number of projects at it over OTLP and their telemetry lands in one place. +## Architecture + +Everything enters through one gateway — the OTel Collector — so a project +configures a single OTLP endpoint and a backend can be swapped without +touching any app. Tempo derives RED metrics from spans, so a service that +only sends traces still gets dashboards and error alerting. It all runs on +one host on purpose: at CML's telemetry volume, distributed ingest would add +operational weight for no gain. Rationale and alternatives: +[ADR 0001](docs/adr/0001-observability-stack.md). + ```mermaid flowchart LR - apps["Project apps
(OTLP gRPC 4317 / HTTP 4318)"] --> otel["OTel Collector
(ingestion gateway)"] - otel -- logs --> loki[Loki] - otel -- traces --> tempo[Tempo] - otel -- metrics --> prom[Prometheus] - tempo -- "span metrics (RED)" --> prom - loki --> grafana[Grafana] - tempo --> grafana - prom --> grafana - grafana --- cf["Cloudflare Tunnel
(optional overlay)"] + apps["Project apps"] -->|"OTLP over HTTPS
(gRPC/HTTP direct on private nets)"| cf["Cloudflare Tunnel
(production, optional)"] + user["Browser"] -->|HTTPS| cf + cf --> otel + cf --> grafana + + subgraph host["Monitoring host — Docker Compose, ports bound to 127.0.0.1"] + otel["OTel Collector
(ingestion gateway)"] + otel -->|logs| loki["Loki"] + otel -->|traces| tempo["Tempo"] + otel -->|metrics| prom["Prometheus"] + tempo -->|"span metrics (RED)"| prom + grafana["Grafana"] -. queries .-> loki & tempo & prom + end ``` +Solid arrows are the write path; dotted arrows are Grafana reading at query +time. Locally (`just up` / `just demo`) there is no tunnel — everything +talks over the compose network and Grafana is on `localhost:3000`. + ## Demo: see it work in one command ```sh @@ -52,9 +70,12 @@ config/ loki.yaml # logs tempo.yaml # traces prometheus.yaml # metrics + alertmanager.yaml # alert routing (webhook + watchdog heartbeat) alerts/ # Prometheus alert rules grafana/ # provisioned datasources + dashboard loader dashboards/ # drop JSON dashboards here; Grafana auto-loads them +docs/ # runbook, onboarding templates, ADRs, screenshots +infra/ # OpenTofu: Cloudflare tunnel, ingress routes, DNS ``` ## Run @@ -67,6 +88,11 @@ just up-tunnel # core stack + Cloudflare Tunnel (needs CLOUDFLARE_TUNNE Grafana: (admin / whatever you set). +The Cloudflare side of the tunnel (hostnames, DNS, the tunnel itself) is +code too: `infra/` holds a small OpenTofu config whose output is the +`CLOUDFLARE_TUNNEL_TOKEN` the overlay needs — bootstrap instructions in +[infra/main.tf](infra/main.tf). + `just check` validates everything (compose files, Prometheus config and alert rules, collector config, YAML, workflows, dashboard JSON) in pinned containers — no host installs. CI runs the same command on every push and @@ -75,71 +101,29 @@ healthy. ## Sending telemetry from a project -### Traces + metrics (OTLP) - -Point your app (or its local OTel Collector) at this host's OTLP endpoints: - -- gRPC: `:4317` -- HTTP: `:4318` - -Do **not** publish those ports to the public internet. Expose the monitoring -host via Cloudflare Tunnel, Tailscale, or WireGuard — the compose file binds -them to `127.0.0.1` to make that the obvious path. - -### Logs (Loki Docker driver) - -Container stdout/stderr is best shipped directly by the Docker daemon using -the Loki log driver plugin. On each project host: - -```sh -# 1. Install the driver plugin (once per host) -docker plugin install grafana/loki-docker-driver:latest \ - --alias loki --grant-all-permissions - -# 2. Set LOKI_URL in the host's .env, e.g. -# LOKI_URL=https://logs.example.org/loki/api/v1/push -``` - -How the project wires it up is project-specific. The RELab repo, for -example, uses an optional overlay (`compose.logging.loki.yml`) that the -justfile auto-includes when `LOKI_URL` is set — hosts without Loki keep -Docker's default json-file driver. Other projects can do the equivalent: -set the daemon-wide `log-driver` in `/etc/docker/daemon.json`, or add a -per-service `logging:` block. +One OTLP endpoint (`:4317` gRPC / `:4318` HTTP), bearer-token auth +(`OTLP_AUTH_TOKEN`), and three naming conventions. Copy-paste templates for +every ingestion form — zero-code Python/FastAPI, plain OTLP env vars, the +Loki Docker driver, and Grafana Alloy for file logs — live in +**[docs/ONBOARDING.md](docs/ONBOARDING.md)**. -Logs carry labels for `service`, `env`, and `host` so you can filter in -Grafana. Keep label cardinality low — don't add `user_id`, `request_id`, -etc. as labels; use LogQL filters for those. +Never publish 4317/4318 directly; the compose file binds them to +`127.0.0.1` and the tunnel is the exposure path. ## Alerting Prometheus evaluates the rules in `config/alerts/` (target down, OTel -export failures, >5% span error rate) and Grafana surfaces them under -Alerting → Alert rules. There is deliberately no Alertmanager: on a -single-host stack, another service buys routing complexity before anyone -needs it. Add one (or a Grafana contact point) when alerts must page -someone. +export failures, >5% span error rate, disk >80%); Alertmanager delivers +them to any webhook via `ALERT_WEBHOOK_URL`, and the always-firing +`Watchdog` posts to `HEARTBEAT_URL` every 5 minutes — point that at a dead +man's switch so you hear about it when the monitoring host itself dies. +Both are optional; unset means alerts are visible in Grafana only. +Operations (rotating tokens, disk pressure, backup/restore): +**[docs/RUNBOOK.md](docs/RUNBOOK.md)**. ## Storage Everything persists to local Docker volumes (`loki_data`, `tempo_data`, `prometheus_data`, `grafana_data`). Swap Loki / Tempo storage to S3-compatible (Backblaze B2, Cloudflare R2, Hetzner, MinIO) when you outgrow local disk — -`compose.storage-s3.yml` documents the concrete shape of that change, and -`docs/adr/0001-observability-stack.md` records the architecture rationale. - -## Design decisions - -Everything enters through one gateway — the collector — so a project -configures a single OTLP endpoint, and a backend can be swapped without -touching any app. It runs on one host on purpose: CML's telemetry is a handful -of services at single-digit requests per second, so distributed ingest would -add operational weight for no gain, while one compose file stays auditable by a -single person. When local disk runs short, S3-backed Loki and Tempo (above) -are the documented way out. - -Logs carry only low-cardinality labels — `service`, `env`, `host` — and -everything else is a query-time filter; high-cardinality labels are the usual -way a Loki install falls over. And because Tempo's metrics generator derives -RED metrics from spans, any service that sends traces gets the Service Health -dashboard whether or not it emits metrics of its own. +`compose.storage-s3.yml` documents the concrete shape of that change. diff --git a/compose.demo.yml b/compose.demo.yml index 0d1d317..16c100f 100644 --- a/compose.demo.yml +++ b/compose.demo.yml @@ -1,23 +1,28 @@ # Demo overlay: a tiny auto-instrumented FastAPI service plus a load # generator, so the stack has real traces, metrics, and correlated logs -# to show. Not for production. +# to show. Not for production — no restart-across-reboots on purpose. # # just demo # core stack + this overlay -# just demo-down +# just demo-down # remove just the demo services + +x-demo-logging: &demo-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" services: demo-api: build: ./demo - restart: unless-stopped + restart: on-failure depends_on: [ otel-collector ] environment: OTEL_SERVICE_NAME: demo-api OTEL_RESOURCE_ATTRIBUTES: env=demo OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf - OTEL_TRACES_EXPORTER: otlp - OTEL_METRICS_EXPORTER: otlp - OTEL_LOGS_EXPORTER: otlp + OTEL_EXPORTER_OTLP_HEADERS: Authorization=Bearer ${OTLP_AUTH_TOKEN:?set OTLP_AUTH_TOKEN in .env} + # Traces/metrics/logs exporters all default to otlp via opentelemetry-distro. # Ship Python log records via OTLP with trace context attached. OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED: "true" OTEL_METRIC_EXPORT_INTERVAL: "10000" @@ -25,15 +30,11 @@ services: # http.route) rather than the legacy names. Stable metric names are frozen, # so the Service Health dashboard's app-SDK panel survives library bumps. OTEL_SEMCONV_STABILITY_OPT_IN: http - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" + logging: *demo-logging demo-load: image: curlimages/curl:8.21.0 - restart: unless-stopped + restart: on-failure depends_on: [ demo-api ] command: - sh @@ -44,8 +45,4 @@ services: curl -s -o /dev/null demo-api:8000/work sleep 1 done - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" + logging: *demo-logging diff --git a/compose.tunnel.yml b/compose.tunnel.yml index 106896e..d39f298 100644 --- a/compose.tunnel.yml +++ b/compose.tunnel.yml @@ -3,10 +3,9 @@ # # just up-tunnel # -# Token-based tunnel: routes are configured in the Cloudflare Zero Trust -# dashboard. Create a tunnel at dash.cloudflare.com → Zero Trust → Networks → -# Tunnels, then set the public hostnames there (e.g. grafana.example.com → -# http://grafana:3000, otlp.example.com → http://otel-collector:4318). +# The tunnel, its public hostnames, and DNS are managed as code in infra/ +# (OpenTofu); `tofu output -raw tunnel_token` yields the token for .env. +# See infra/main.tf for the bootstrap steps. services: cloudflared: diff --git a/compose.yml b/compose.yml index 454c22f..43238ee 100644 --- a/compose.yml +++ b/compose.yml @@ -29,8 +29,9 @@ services: volumes: - ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro command: [ "--config=/etc/otelcol/config.yaml" ] + environment: + OTLP_AUTH_TOKEN: ${OTLP_AUTH_TOKEN:?set OTLP_AUTH_TOKEN in .env} logging: *default-logging - platform: linux/amd64 loki: image: grafana/loki:3.7.3 @@ -42,7 +43,11 @@ services: logging: *default-logging tempo: - image: grafana/tempo:3.0.2 + # Held at 2.x: Tempo 3.0 is a re-architecture (ingester/compactor removed, + # metrics-generator changed) needing a real config migration — see + # https://grafana.com/docs/tempo/latest/set-up-for-tracing/setup-tempo/migrate-to-3/ + # Dependabot ignores 3.x majors until that migration is done deliberately. + image: grafana/tempo:2.10.4 restart: unless-stopped volumes: - ./config/tempo.yaml:/etc/tempo/tempo.yaml:ro @@ -65,6 +70,42 @@ services: # Prometheus 3.x: OTLP ingestion is its own flag, no longer a feature flag. - --web.enable-otlp-receiver - --enable-feature=native-histograms,exemplar-storage + # Whichever hits first wins; the disk-space alert is the backstop. + - --storage.tsdb.retention.size=15GB + logging: *default-logging + + alertmanager: + image: prom/alertmanager:v0.33.0 + restart: unless-stopped + volumes: + - ./config/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml:ro + - alertmanager_data:/alertmanager + environment: + ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-} + HEARTBEAT_URL: ${HEARTBEAT_URL:-} + # Alertmanager can't expand env vars in its config; write the webhook + # URLs to the url_file paths the config points at, then start. tmpfs on + # purpose: the URLs often carry secrets and must not end up in the + # persistent volume (which `just backup` archives). + tmpfs: [ /run/am ] + entrypoint: [ "/bin/sh", "-c" ] + command: + - | + printf '%s' "$$ALERT_WEBHOOK_URL" > /run/am/webhook_url + printf '%s' "$$HEARTBEAT_URL" > /run/am/heartbeat_url + exec /bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yaml --storage.path=/alertmanager + logging: *default-logging + + node-exporter: + image: prom/node-exporter:v1.11.1 + restart: unless-stopped + command: [ "--path.rootfs=/host" ] + pid: host + volumes: + # No rslave: unsupported on Docker Desktop; only affects mounts added after start. + # No network_mode:host either, so node_network_* describes the container + # veth, not host NICs — fine while nothing alerts on network metrics. + - /:/host:ro logging: *default-logging grafana: @@ -82,7 +123,6 @@ services: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} GF_USERS_ALLOW_SIGN_UP: "false" GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000} - GF_FEATURE_TOGGLES_ENABLE: traceqlEditor,metricsSummary,traceToProfiles logging: *default-logging volumes: @@ -90,3 +130,4 @@ volumes: tempo_data: prometheus_data: grafana_data: + alertmanager_data: diff --git a/config/alertmanager.yaml b/config/alertmanager.yaml new file mode 100644 index 0000000..72b3289 --- /dev/null +++ b/config/alertmanager.yaml @@ -0,0 +1,27 @@ +# Alert routing. Webhook URLs are supplied at runtime via url_file (written +# from ALERT_WEBHOOK_URL / HEARTBEAT_URL by the compose entrypoint) because +# Alertmanager does not expand env vars in its config. With the env vars +# unset, delivery fails with a logged error and nothing else — the stack +# runs fine without them. + +route: + receiver: webhook + group_by: [alertname] + routes: + # Watchdog is always firing; its delivery is the heartbeat. Wire + # HEARTBEAT_URL to a dead man's switch (e.g. healthchecks.io) that + # alerts when pings STOP arriving. + - matchers: [ 'alertname = "Watchdog"' ] + receiver: heartbeat + group_wait: 0s + repeat_interval: 5m + +receivers: + - name: webhook + webhook_configs: + - url_file: /run/am/webhook_url + + - name: heartbeat + webhook_configs: + - url_file: /run/am/heartbeat_url + send_resolved: false diff --git a/config/alerts/stack.yaml b/config/alerts/stack.yaml index c8016e3..07d80bd 100644 --- a/config/alerts/stack.yaml +++ b/config/alerts/stack.yaml @@ -1,10 +1,22 @@ # Prometheus alert rules for the stack itself plus RED-style alerts on the -# span metrics Tempo derives from traces. Evaluated by Prometheus; visible in -# Grafana under Alerting → Alert rules (Prometheus datasource). -# No Alertmanager: single-host stack, alerts are surfaced in Grafana. -# Validated by `just check` (promtool check rules). +# span metrics Tempo derives from traces. Evaluated by Prometheus, delivered +# by Alertmanager (config/alertmanager.yaml → ALERT_WEBHOOK_URL/HEARTBEAT_URL), +# and visible in Grafana under Alerting → Alert rules. +# Validated by `just check` (promtool check config, which loads these rules). groups: + - name: meta + rules: + # Always firing on purpose: its arrival at the heartbeat receiver is + # the proof the whole alerting pipeline works. Silence = broken. + - alert: Watchdog + expr: vector(1) + labels: + severity: none + annotations: + summary: "Alerting-pipeline heartbeat (always firing)" + description: "Route this to a dead man's switch; investigate if pings stop." + - name: stack-health rules: - alert: TargetDown @@ -17,9 +29,9 @@ groups: description: "Prometheus cannot scrape {{ $labels.instance }} (job {{ $labels.job }}) for 2 minutes." - alert: OtelExportFailures - expr: sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m])) > 0 - or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points_total[5m])) > 0 - or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m])) > 0 + expr: sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m])) > 0 + or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[5m])) > 0 + or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m])) > 0 for: 5m labels: severity: warning @@ -27,6 +39,19 @@ groups: summary: "OTel Collector failing to export via {{ $labels.exporter }}" description: "The collector has been failing to deliver telemetry to a backend for 5 minutes. Check `just logs otel-collector`." + - name: capacity + rules: + - alert: HostDiskSpaceLow + expr: > + (node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|overlay"} + / node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|overlay"}) < 0.2 + for: 15m + labels: + severity: warning + annotations: + summary: "Filesystem {{ $labels.mountpoint }} is over 80% full" + description: "Loki and Tempo have no total-size cap, so this alert is the storage backstop. Free space or lower retention (see RUNBOOK)." + - name: service-red rules: - alert: HighErrorRate diff --git a/config/otel-collector.yaml b/config/otel-collector.yaml index 9350cb9..338ab99 100644 --- a/config/otel-collector.yaml +++ b/config/otel-collector.yaml @@ -2,13 +2,23 @@ # appropriate backend. Keep this config dumb — do shaping/enrichment at the # app edge (per-project collector) where it has context. +# All senders must present "Authorization: Bearer ". The +# OTLP hostnames are public behind the tunnel, so ingestion needs auth. +extensions: + bearertokenauth: + token: ${env:OTLP_AUTH_TOKEN} + receivers: otlp: protocols: grpc: endpoint: 0.0.0.0:4317 + auth: + authenticator: bearertokenauth http: endpoint: 0.0.0.0:4318 + auth: + authenticator: bearertokenauth processors: memory_limiter: @@ -34,6 +44,7 @@ exporters: endpoint: http://prometheus:9090/api/v1/otlp service: + extensions: [bearertokenauth] pipelines: logs: receivers: [otlp] diff --git a/config/prometheus.yaml b/config/prometheus.yaml index 1d4c137..5df8860 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -7,6 +7,11 @@ global: rule_files: - /etc/prometheus/alerts/*.yaml +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + # Apps push metrics via OTLP (through the collector). Scrape only infra we host. scrape_configs: - job_name: prometheus @@ -16,3 +21,7 @@ scrape_configs: - job_name: otel-collector static_configs: - targets: ["otel-collector:8888"] + + - job_name: node + static_configs: + - targets: ["node-exporter:9100"] diff --git a/config/tempo.yaml b/config/tempo.yaml index 4f40301..ba7b78b 100644 --- a/config/tempo.yaml +++ b/config/tempo.yaml @@ -8,10 +8,9 @@ distributor: receivers: otlp: protocols: + # gRPC only: the collector is the sole sender (otlp_grpc/tempo). grpc: endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 ingester: max_block_duration: 5m diff --git a/dashboards/.gitkeep b/dashboards/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/dashboards/stack-health.json b/dashboards/stack-health.json new file mode 100644 index 0000000..c8f74cf --- /dev/null +++ b/dashboards/stack-health.json @@ -0,0 +1,139 @@ +{ + "title": "Stack Health", + "uid": "stack-health", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-3h", "to": "now" }, + "graphTooltip": 1, + "tags": ["infra"], + "panels": [ + { + "type": "timeseries", + "title": "Filesystem Used", + "description": "Loki and Tempo have no total-size cap — the HostDiskSpaceLow alert fires at 80%.", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, + "id": 1, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|ramfs|overlay\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|ramfs|overlay\"})", + "legendFormat": "{{mountpoint}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Host CPU", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, + "id": 2, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "legendFormat": "used", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Host Memory Used", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, + "id": 3, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", + "legendFormat": "used", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Collector Ingest Rate", + "description": "Items accepted by the OTLP receivers, per signal.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 4, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", + "legendFormat": "spans/s", + "refId": "A" + }, + { + "expr": "sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", + "legendFormat": "metric points/s", + "refId": "B" + }, + { + "expr": "sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", + "legendFormat": "log records/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Collector Export Failures", + "description": "Anything above zero means a backend is rejecting or unreachable — the OtelExportFailures alert covers this.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 5, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval])) or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval])) or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", + "legendFormat": "{{exporter}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0, + "color": { "mode": "fixed", "fixedColor": "red" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [] + } + } + ] +} diff --git a/demo/Dockerfile b/demo/Dockerfile index d3c9bee..c55926a 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -1,11 +1,7 @@ FROM python:3.13-slim -RUN pip install --no-cache-dir \ - fastapi==0.139.0 \ - uvicorn==0.49.0 \ - opentelemetry-distro==0.64b0 \ - opentelemetry-exporter-otlp==1.43.0 \ - opentelemetry-instrumentation-fastapi==0.64b0 +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt WORKDIR /app COPY app.py . diff --git a/demo/requirements.txt b/demo/requirements.txt new file mode 100644 index 0000000..3815ed8 --- /dev/null +++ b/demo/requirements.txt @@ -0,0 +1,5 @@ +fastapi==0.139.0 +uvicorn==0.49.0 +opentelemetry-distro==0.64b0 +opentelemetry-exporter-otlp==1.43.0 +opentelemetry-instrumentation-fastapi==0.64b0 diff --git a/docs/ONBOARDING.md b/docs/ONBOARDING.md new file mode 100644 index 0000000..c1705ae --- /dev/null +++ b/docs/ONBOARDING.md @@ -0,0 +1,123 @@ +# Sending telemetry from your project + +One endpoint, one token, three conventions — then pick the template that +matches how your project runs. + +## The endpoint + +| | | +| --- | --- | +| Production (via tunnel) | `https://otlp.` — OTLP **HTTP** (`http/protobuf`) only | +| Private network / same host | `:4317` (gRPC) or `:4318` (HTTP) | +| Auth | `Authorization: Bearer ` (ask the stack operator) | + +The tunnel routes only HTTPS to the collector's HTTP receiver — there is no +public gRPC path, so set `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` when +sending through it. gRPC works only on private paths (VPN/WireGuard, same +Docker network). Never expose 4317/4318 directly. + +## The conventions + +- **`service.name`** — required; one stable name per deployable unit + (`relab-api`, not `relab-api-prod-2`). This is what dashboards key on. +- **`env`** — `prod`, `staging`, `dev`, set via resource attributes. +- **Keep label cardinality low** — no user IDs, request IDs, or timestamps + in resource attributes or log labels; those belong in log lines and span + attributes, where they're query-time filters. + +Any service that sends **traces** automatically gets RED metrics, the +Service Health dashboard, and error-rate alerting — Tempo derives them from +spans. Send traces first; everything else is a bonus. + +## Template 1 — Python/FastAPI, zero code changes + +```sh +pip install opentelemetry-distro opentelemetry-exporter-otlp opentelemetry-instrumentation-fastapi +``` + +```sh +export OTEL_SERVICE_NAME=my-service +export OTEL_RESOURCE_ATTRIBUTES=env=prod +export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.example.org +export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer " +export OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true +export OTEL_SEMCONV_STABILITY_OPT_IN=http + +opentelemetry-instrument uvicorn app:app --host 0.0.0.0 --port 8000 +``` + +Traces, RED metrics, and logs with trace context — no OTel code in the app. +The living example is this repo's own [`demo/`](../demo/) + +[`compose.demo.yml`](../compose.demo.yml). + +## Template 2 — any language, plain OTLP + +Every OTel SDK understands the same four env vars: + +```sh +OTEL_SERVICE_NAME=my-service +OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.example.org +OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer " +OTEL_RESOURCE_ATTRIBUTES=env=prod +``` + +## Template 3 — Docker container logs (Loki driver) + +For shipping container stdout/stderr without touching the app. Requires a +Loki push URL, which **this stack does not expose by default** — Loki has no +auth, so a push hostname must first be added to `infra/main.tf` and protected +(Cloudflare Access service token), or reached over a private network path +(VPN/WireGuard). If in doubt, use the OTLP log path above instead. + +```sh +# once per host +docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions +``` + +```yaml +# per service, in compose.yml +logging: + driver: loki + options: + loki-url: ${LOKI_URL} # e.g. https://logs.example.org/loki/api/v1/push + loki-external-labels: service={{.Name}},env=prod,host=myhost +``` + +RELab's `compose.logging.loki.yml` overlay (auto-included when `LOKI_URL` +is set) is the reference implementation of this pattern. + +## Template 4 — host or file logs (Grafana Alloy) + +For log files outside containers (Promtail is EOL; Alloy is its successor): + +```alloy +// alloy/config.alloy +local.file_match "app" { + path_targets = [{ __path__ = "/var/log/myapp/*.log", service = "myapp", env = "prod", host = "myhost" }] +} + +loki.source.file "app" { + targets = local.file_match.app.targets + forward_to = [loki.write.central.receiver] +} + +loki.write "central" { + endpoint { + url = "https://logs.example.org/loki/api/v1/push" + } +} +``` + +```yaml +# compose service +alloy: + image: grafana/alloy:v1.13.0 + restart: unless-stopped + command: [ "run", "/etc/alloy/config.alloy" ] + volumes: + - ./alloy/config.alloy:/etc/alloy/config.alloy:ro + - /var/log/myapp:/var/log/myapp:ro +``` + +Same caveat as Template 3: the Loki push URL needs a protected network path. diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md new file mode 100644 index 0000000..4d1624a --- /dev/null +++ b/docs/RUNBOOK.md @@ -0,0 +1,92 @@ +# Runbook + +Operational procedures for this stack. Commands run from the repo root on +the monitoring host. `just ps` and the **Stack Health** dashboard are the +first stop for anything. + +![Stack Health dashboard](img/stack-health.png) + +(That capture is honest: the red export-failure spike is a Tempo outage +after a bad major-version bump, and the ingest gap is a backup/restore +drill — both procedures below.) + +## A service is down / misbehaving + +```sh +just ps # what's running, what's restarting +just logs # follow logs (otel-collector, loki, tempo, prometheus, grafana, alertmanager) +just restart +``` + +`TargetDown` fires after 2 minutes if Prometheus can't scrape the collector, +node-exporter, or itself. `OtelExportFailures` means the collector is up but +a backend is rejecting data — check that backend's logs. + +## Disk filling up (`HostDiskSpaceLow`) + +Retention is only partially size-bounded by design: + +| Data | Time limit | Size limit | +| --- | --- | --- | +| Container stdout logs | — | json-file 10m × 3 per service | +| Prometheus TSDB | 30d | 15GB (`--storage.tsdb.retention.size`) | +| Loki chunks | 30d | none — Loki cannot cap total size | +| Tempo blocks | 7d | none | + +So when the 80% alert fires: check the Filesystem panel on Stack Health, +then either free space or shorten retention — `retention_period` in +`config/loki.yaml`, `block_retention` in `config/tempo.yaml`, the +`--storage.tsdb.retention.*` flags in `compose.yml` — and restart the +affected service. If disk pressure is chronic, move Loki/Tempo to object +storage: `compose.storage-s3.yml`. + +## Backup and restore + +```sh +just backup # pauses stateful services for seconds, writes backups/monitoring-.tar.gz +just restore backups/monitoring-.tar.gz # stops the stack, wipes volumes, restores +just up +``` + +Backups are crash-consistent (equivalent to a power loss; every component +recovers via its WAL). Tarballs are mode 0600 and contain secrets (Grafana +DB); copy them off-host over a private channel — a backup on the disk it +protects is a decoration. RPO = however often you run it; a daily cron on +the host is the intended setup. During the pause the collector keeps +accepting telemetry and retries for ~5 minutes; a backup that takes longer +than that will drop data, so on large volumes run it at a quiet hour. + +## Rotating secrets + +- **OTLP token:** new value for `OTLP_AUTH_TOKEN` in `.env` → + `docker compose up -d otel-collector` → update every sender's + `OTEL_EXPORTER_OTLP_HEADERS`. Senders with the old token get 401s (visible + as their export errors) until updated. That includes a running demo + overlay — re-run `just demo` to recreate it with the new token. +- **Tunnel token:** rotate in Cloudflare Zero Trust → new + `CLOUDFLARE_TUNNEL_TOKEN` in `.env` → `just up-tunnel`. +- **Grafana admin password:** `GRAFANA_ADMIN_PASSWORD` in `.env` → + `docker compose up -d grafana`. + +## Alert delivery + +Prometheus evaluates rules → Alertmanager delivers: + +- `ALERT_WEBHOOK_URL` — all alerts (any webhook receiver: ntfy, Slack, …). +- `HEARTBEAT_URL` — the always-firing `Watchdog` posts here every 5 minutes. + Point it at a dead man's switch (e.g. healthchecks.io) that alerts when + pings **stop**: that is the "monitoring host is dead" signal nothing + inside the host can send. + +Both empty = no delivery; Alertmanager logs a notify error per cycle, which +is expected and harmless. After changing either, `docker compose up -d +alertmanager`. + +## Upgrading images + +Dependabot PRs bump the pins. For each: CI runs `just check`; after merge, +on the host: `git pull && just pull && just up`. The `just check` validators +(promtool, otelcol, amtool) read their image versions from `compose.yml`, so +every bump is validated with the exact binaries the stack will run — CI +fails loudly when config syntax drifts between versions, which is the point. +Tempo majors are deliberately held back (see the comment in `compose.yml`). diff --git a/docs/img/stack-health.png b/docs/img/stack-health.png new file mode 100644 index 0000000..ac68421 Binary files /dev/null and b/docs/img/stack-health.png differ diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 0000000..95024a4 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/cloudflare/cloudflare" { + version = "5.21.1" + constraints = "~> 5.0" + hashes = [ + "h1:0mY28ndURk1ft6Xgh+BJ8v6hM0+KQqWYS32NAJ/ffMo=", + "h1:2gZZ/mcfc1aQmgiFpQ5JH69tamBLO76Iw8TaQZpeWM4=", + "h1:6m7m8cPUwqoCldfIxJRCYdhHTZvdIPwtUObaPKWnZH8=", + "h1:JxR20UmUG+tZoocOFfOHy1bpd7IiC2lyvxJU45U9KEY=", + "h1:Lh5LHSNKoKwCx4F/YlRjoLZ+jZLxnoxEfOMzUX9n4zg=", + "h1:gNF1Sro3G9nXhtdkitXwDVKxI1jpBAf8KPv+Y4kAJwk=", + "h1:hU72otEs26Wx6tcJD9igX6I/BQtVgeRuaIe3s/hn6bQ=", + "h1:iWJb0lHfVWmCJQSyroXOT8zQlFOT8k1caHcfaooG5wk=", + "zh:049719425b8be43d9d4f0c208217aca0baa22374f061d7ff92f02563490f649c", + "zh:0a8a3c1b26680b437fe9e7910ca81e532d36f8efacfb14f45690b6a779856993", + "zh:32b61f80892243f7ab8e453fa038c1f3e2aac733ccb98307c2cfe798b2793b32", + "zh:42c27f3cd62979e70716c51f682a3d131d51ad76d86dff83d8cdbfffcebac841", + "zh:4c8cd464f9b6ecde5cd4430bbba4be3b810826105e51ef6328b6a2b69f821443", + "zh:586ea42ef74d6c5bc4c9b89da6b1f8618a19f4e80272fe8d615e7d5b11c491af", + "zh:b09b86c7cac7085e01c9b7a828f09d13c44589d3e3cd42f0b694ca3e4cd3ed0a", + "zh:eac80665e60c701b37a6318f4e405d67f1720f8da5f93135c6256049282d3367", + "zh:f809ab383cca0a5f83072981c64208cbd7fa67e986a86ee02dd2c82333221e32", + ] +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 0000000..20fbd5c --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,98 @@ +# Cloudflare edge for the monitoring stack: the tunnel, its ingress rules, +# and DNS. This is the only part of the stack that otherwise lives as +# click-ops in the Zero Trust dashboard. +# +# Bootstrap (owner-run, once): +# export CLOUDFLARE_API_TOKEN=... # needs Tunnel:Edit, DNS:Edit +# cd infra && tofu init && tofu apply +# tofu output -raw tunnel_token # → CLOUDFLARE_TUNNEL_TOKEN in ../.env +# +# State is local (infra/terraform.tfstate, gitignored) — one host, one +# operator; move it to R2 the day a second operator exists. + +terraform { + required_version = ">= 1.8" + required_providers { + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 5.0" + } + } +} + +provider "cloudflare" { + # Auth via the CLOUDFLARE_API_TOKEN environment variable. +} + +variable "account_id" { + type = string + description = "Cloudflare account ID (dash.cloudflare.com → overview sidebar)." +} + +variable "zone_id" { + type = string + description = "Zone ID of the domain the hostnames live under." +} + +variable "domain" { + type = string + description = "Apex domain, e.g. example.org → grafana.example.org, otlp.example.org." +} + +resource "cloudflare_zero_trust_tunnel_cloudflared" "monitoring" { + account_id = var.account_id + name = "monitoring" + config_src = "cloudflare" +} + +resource "cloudflare_zero_trust_tunnel_cloudflared_config" "monitoring" { + account_id = var.account_id + tunnel_id = cloudflare_zero_trust_tunnel_cloudflared.monitoring.id + + config = { + ingress = [ + { + hostname = "grafana.${var.domain}" + service = "http://grafana:3000" + }, + { + # OTLP HTTP ingestion; the collector enforces bearer-token auth. + hostname = "otlp.${var.domain}" + service = "http://otel-collector:4318" + }, + { + service = "http_status:404" + }, + ] + } +} + +resource "cloudflare_dns_record" "grafana" { + zone_id = var.zone_id + # Provider v5 requires the full hostname (terraform-provider-cloudflare#5620). + name = "grafana.${var.domain}" + type = "CNAME" + content = "${cloudflare_zero_trust_tunnel_cloudflared.monitoring.id}.cfargotunnel.com" + proxied = true + ttl = 1 +} + +resource "cloudflare_dns_record" "otlp" { + zone_id = var.zone_id + name = "otlp.${var.domain}" + type = "CNAME" + content = "${cloudflare_zero_trust_tunnel_cloudflared.monitoring.id}.cfargotunnel.com" + proxied = true + ttl = 1 +} + +data "cloudflare_zero_trust_tunnel_cloudflared_token" "monitoring" { + account_id = var.account_id + tunnel_id = cloudflare_zero_trust_tunnel_cloudflared.monitoring.id +} + +output "tunnel_token" { + description = "Set as CLOUDFLARE_TUNNEL_TOKEN in ../.env for just up-tunnel." + value = data.cloudflare_zero_trust_tunnel_cloudflared_token.monitoring.token + sensitive = true +} diff --git a/justfile b/justfile index 61116ac..7b6ce64 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,10 @@ set dotenv-load +# Stateful services and their volumes, shared by backup/restore. The volume +# names assume the compose project name "monitoring" (see guard in backup). +stateful := "grafana prometheus loki tempo alertmanager" +backup_mounts := "-v monitoring_grafana_data:/data/grafana -v monitoring_prometheus_data:/data/prometheus -v monitoring_loki_data:/data/loki -v monitoring_tempo_data:/data/tempo -v monitoring_alertmanager_data:/data/alertmanager" + default: @just --list @@ -7,8 +12,11 @@ default: up: docker compose up -d -# Core stack + Cloudflare Tunnel (production; needs CLOUDFLARE_TUNNEL_TOKEN) +# Core stack + Cloudflare Tunnel (production; needs CLOUDFLARE_TUNNEL_TOKEN). +# Refuses to expose the stack with the documented default credentials. up-tunnel: + @[ "${OTLP_AUTH_TOKEN:-}" != "local-dev-token" ] || { echo "error: OTLP_AUTH_TOKEN is still the local default; generate one (openssl rand -hex 32) before exposing ingestion" >&2; exit 1; } + @[ "${GRAFANA_ADMIN_PASSWORD:-}" != "change-me" ] || { echo "error: GRAFANA_ADMIN_PASSWORD is still the documented default; change it before exposing Grafana" >&2; exit 1; } docker compose -f compose.yml -f compose.tunnel.yml up -d down: @@ -19,8 +27,9 @@ down: demo: docker compose -f compose.yml -f compose.demo.yml up -d --build +# Remove only the demo services; the core stack keeps running. demo-down: - docker compose -f compose.yml -f compose.demo.yml down + docker compose -f compose.yml -f compose.demo.yml rm -sf demo-api demo-load logs service="": docker compose logs -f {{service}} @@ -39,22 +48,51 @@ tail service: docker compose logs -f --no-log-prefix {{service}} | jq -R 'fromjson? // .' # Validate everything. All validators run in containers — no host installs. +# promtool/otelcol/amtool images are read from compose.yml so they can't +# drift from the versions the stack actually runs. check: docker compose config -q CLOUDFLARE_TUNNEL_TOKEN=dummy docker compose -f compose.yml -f compose.tunnel.yml config -q docker compose -f compose.yml -f compose.demo.yml config -q - docker run --rm -v ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro -v ./config/alerts:/etc/prometheus/alerts:ro --entrypoint promtool prom/prometheus:v3.11.2 check config /etc/prometheus/prometheus.yaml - docker run --rm -v ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro otel/opentelemetry-collector-contrib:0.150.1 validate --config=/etc/otelcol/config.yaml - docker run --rm -v .:/code:ro pipelinecomponents/yamllint:0.35.13 yamllint -d relaxed . + docker run --rm -v ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro -v ./config/alerts:/etc/prometheus/alerts:ro --entrypoint promtool $(docker compose config --images | grep prom/prometheus) check config /etc/prometheus/prometheus.yaml + docker run --rm -e OTLP_AUTH_TOKEN=dummy -v ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro $(docker compose config --images | grep opentelemetry-collector) validate --config=/etc/otelcol/config.yaml + docker run --rm -v ./config/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml:ro --entrypoint /bin/amtool $(docker compose config --images | grep prom/alertmanager) check-config /etc/alertmanager/alertmanager.yaml + docker run --rm -v .:/code:ro pipelinecomponents/yamllint:0.35.13 yamllint -d '{extends: relaxed, ignore: [.git/, backups/, infra/.terraform/]}' . docker run --rm -v .:/repo:ro -w /repo rhysd/actionlint:1.7.12 -color - jq empty dashboards/*.json + docker run --rm -v ./infra:/infra:ro -w /infra ghcr.io/opentofu/opentofu:1.12.3 fmt -check + docker run --rm -v ./dashboards:/dashboards:ro ghcr.io/jqlang/jq:1.8.1 empty $(ls dashboards/*.json | sed 's|^dashboards|/dashboards|') + +# Full OpenTofu validation (downloads the provider, so not part of `check`). +infra-validate: + docker run --rm --entrypoint sh -v ./infra:/infra -w /infra ghcr.io/opentofu/opentofu:1.12.3 -c 'tofu init -backend=false -input=false >/dev/null && tofu validate' # Format YAML in place (needs yamlfmt on the host; optional). fmt: yamlfmt . -# Boot the core stack and wait until Grafana reports healthy. Used by CI. +# Snapshot all stateful volumes to backups/.tar.gz (mode 0600 — +# it contains the Grafana DB and webhook secrets; copy it off-host, keep it +# private). Services are paused during the copy, and unpaused even if the +# copy fails. +backup: + @[ -z "${COMPOSE_PROJECT_NAME:-}" ] || { echo "error: COMPOSE_PROJECT_NAME is set; backup expects the monitoring_* volume names" >&2; exit 1; } + mkdir -p backups + -docker compose unpause {{stateful}} 2>/dev/null + docker compose pause {{stateful}} && { docker run --rm {{backup_mounts}} -v ./backups:/backups alpine:3.24 sh -c 'umask 077 && tar czf /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz -C /data .'; rc=$?; docker compose unpause {{stateful}}; exit $rc; } + @ls -lh backups/ | tail -1 + +# Restore a backup tarball into the volumes (stops the stack; wipes current state). +restore file: + @[ -f "{{file}}" ] || { echo "error: {{file}} not found" >&2; exit 1; } + docker run --rm -v {{absolute_path(file)}}:/backup.tar.gz:ro alpine:3.24 tar tzf /backup.tar.gz > /dev/null + docker compose down --remove-orphans + docker run --rm {{backup_mounts}} -v {{absolute_path(file)}}:/backup.tar.gz:ro alpine:3.24 sh -c 'for d in /data/*; do find "$d" -mindepth 1 -delete; done && tar xzf /backup.tar.gz -C /data' + @echo "Restored {{file}} — run 'just up' to start the stack." + +# Boot the core stack, wait until Grafana reports healthy, and fail if any +# service is crash-looping. Used by CI. smoke: docker compose up -d n=0; until curl -sf http://localhost:3000/api/health >/dev/null; do n=$((n+3)); [ $n -ge 120 ] && { echo "Grafana not healthy after 120s" >&2; exit 1; }; sleep 3; done - @echo "Grafana healthy" + @[ -z "$(docker compose ps -q --status=restarting --status=exited)" ] || { echo "error: services not running:" >&2; docker compose ps >&2; exit 1; } + @echo "Stack healthy"