feat: add Prometheus alert rules

Target-down, OTel export failures, and >5% span error rate (from Tempo
span metrics). Evaluated by Prometheus, surfaced in Grafana Alerting; no
Alertmanager to keep the stack minimal.

Author: simonvanlierde

compose.yml

@@ -55,6 +55,7 @@ services:
     restart: unless-stopped
     volumes:
       - ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
+      - ./config/alerts:/etc/prometheus/alerts:ro
       - prometheus_data:/prometheus
     command:
       - --config.file=/etc/prometheus/prometheus.yaml

config/alerts/stack.yaml

@@ -0,0 +1,41 @@
+# Prometheus alert rules for the stack itself plus RED-style alerts on the
+# span metrics Tempo derives from traces. Evaluated by Prometheus; visible in
+# Grafana under Alerting → Alert rules (Prometheus datasource).
+# No Alertmanager: single-host stack, alerts are surfaced in Grafana.
+# Validated by `just check` (promtool check rules).
+
+groups:
+  - name: stack-health
+    rules:
+      - alert: TargetDown
+        expr: up == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Scrape target {{ $labels.job }} is down"
+          description: "Prometheus cannot scrape {{ $labels.instance }} (job {{ $labels.job }}) for 2 minutes."
+
+      - alert: OtelExportFailures
+        expr: sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m])) > 0
+          or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points_total[5m])) > 0
+          or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m])) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "OTel Collector failing to export via {{ $labels.exporter }}"
+          description: "The collector has been failing to deliver telemetry to a backend for 5 minutes. Check `just logs otel-collector`."
+
+  - name: service-red
+    rules:
+      - alert: HighErrorRate
+        expr: >
+          sum by (service) (rate(traces_spanmetrics_calls_total{status_code="STATUS_CODE_ERROR"}[5m]))
+          / sum by (service) (rate(traces_spanmetrics_calls_total[5m])) > 0.05
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.service }} error rate above 5%"
+          description: "More than 5% of spans from {{ $labels.service }} report errors (from Tempo span metrics)."

config/prometheus.yaml

@@ -4,6 +4,9 @@ global:
   external_labels:
     origin: monitoring-host
 
+rule_files:
+  - /etc/prometheus/alerts/*.yaml
+
 # Apps push metrics via OTLP (through the collector). Scrape only infra we host.
 scrape_configs:
   - job_name: prometheus