Skip to content

Commit b9b9353

Browse files
simonvanlierdesimonvanlierde
committed
feat: add provisioned Service Health (RED) dashboard
Request rate, error rate, and latency quantiles from Tempo span metrics (with exemplars linking to traces), p95 from the service's own OTLP SDK metrics, and a correlated logs panel. Works for any service that sends traces, selected via a service variable. Demo app now logs at INFO so non-error lines ship too.
1 parent 15124d6 commit b9b9353

2 files changed

Lines changed: 154 additions & 0 deletions

File tree

‎dashboards/service-health.json‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
{
2+
"title": "Service Health (RED)",
3+
"uid": "service-health",
4+
"schemaVersion": 39,
5+
"version": 1,
6+
"refresh": "30s",
7+
"time": { "from": "now-1h", "to": "now" },
8+
"graphTooltip": 1,
9+
"tags": ["red", "otlp"],
10+
"templating": {
11+
"list": [
12+
{
13+
"name": "service",
14+
"label": "Service",
15+
"type": "query",
16+
"datasource": { "type": "prometheus", "uid": "prometheus" },
17+
"definition": "label_values(traces_spanmetrics_calls_total, service)",
18+
"query": { "qryType": 1, "query": "label_values(traces_spanmetrics_calls_total, service)", "refId": "var" },
19+
"refresh": 2,
20+
"sort": 1,
21+
"current": { "text": "demo-api", "value": "demo-api" },
22+
"options": [],
23+
"hide": 0
24+
}
25+
]
26+
},
27+
"panels": [
28+
{
29+
"type": "timeseries",
30+
"title": "Request Rate by Endpoint",
31+
"description": "Server spans per second, from Tempo span metrics — works for any service that sends traces.",
32+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
33+
"id": 1,
34+
"datasource": { "type": "prometheus", "uid": "prometheus" },
35+
"targets": [
36+
{
37+
"expr": "sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))",
38+
"legendFormat": "{{span_name}}",
39+
"refId": "A"
40+
}
41+
],
42+
"fieldConfig": {
43+
"defaults": {
44+
"unit": "reqps",
45+
"custom": { "lineWidth": 2, "fillOpacity": 10 }
46+
},
47+
"overrides": []
48+
}
49+
},
50+
{
51+
"type": "timeseries",
52+
"title": "Error Rate",
53+
"description": "Share of server spans with STATUS_CODE_ERROR.",
54+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
55+
"id": 2,
56+
"datasource": { "type": "prometheus", "uid": "prometheus" },
57+
"targets": [
58+
{
59+
"expr": "100 * sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))",
60+
"legendFormat": "errors",
61+
"refId": "A"
62+
}
63+
],
64+
"fieldConfig": {
65+
"defaults": {
66+
"unit": "percent",
67+
"min": 0,
68+
"color": { "mode": "fixed", "fixedColor": "red" },
69+
"custom": { "lineWidth": 2, "fillOpacity": 10 },
70+
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }] }
71+
},
72+
"overrides": []
73+
}
74+
},
75+
{
76+
"type": "timeseries",
77+
"title": "Latency (from traces, with exemplars)",
78+
"description": "Quantiles over Tempo span-metrics latency. Dots are exemplars — click one to open the exact trace in Tempo.",
79+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
80+
"id": 3,
81+
"datasource": { "type": "prometheus", "uid": "prometheus" },
82+
"targets": [
83+
{
84+
"expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
85+
"legendFormat": "p50",
86+
"refId": "A",
87+
"exemplar": true
88+
},
89+
{
90+
"expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
91+
"legendFormat": "p95",
92+
"refId": "B",
93+
"exemplar": true
94+
},
95+
{
96+
"expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
97+
"legendFormat": "p99",
98+
"refId": "C",
99+
"exemplar": true
100+
}
101+
],
102+
"fieldConfig": {
103+
"defaults": {
104+
"unit": "s",
105+
"custom": { "lineWidth": 2, "fillOpacity": 5 }
106+
},
107+
"overrides": []
108+
}
109+
},
110+
{
111+
"type": "timeseries",
112+
"title": "HTTP Server Duration p95 (app SDK metrics)",
113+
"description": "From the service's own OTLP metrics (OTel SDK), i.e. the collector → Prometheus OTLP path.",
114+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
115+
"id": 4,
116+
"datasource": { "type": "prometheus", "uid": "prometheus" },
117+
"targets": [
118+
{
119+
"expr": "histogram_quantile(0.95, sum by (le, http_target) (rate(http_server_duration_milliseconds_bucket{job=\"$service\"}[$__rate_interval])))",
120+
"legendFormat": "{{http_target}}",
121+
"refId": "A"
122+
}
123+
],
124+
"fieldConfig": {
125+
"defaults": {
126+
"unit": "ms",
127+
"custom": { "lineWidth": 2, "fillOpacity": 10 }
128+
},
129+
"overrides": []
130+
}
131+
},
132+
{
133+
"type": "logs",
134+
"title": "Logs",
135+
"description": "OTLP logs for the service. Expand a line and follow trace_id to jump to the trace in Tempo.",
136+
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
137+
"id": 5,
138+
"datasource": { "type": "loki", "uid": "loki" },
139+
"targets": [
140+
{
141+
"expr": "{service_name=\"$service\"}",
142+
"refId": "A"
143+
}
144+
],
145+
"options": {
146+
"showTime": true,
147+
"wrapLogMessage": true,
148+
"enableLogDetails": true,
149+
"sortOrder": "Descending"
150+
}
151+
}
152+
]
153+
}

‎demo/app.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
from fastapi import FastAPI, HTTPException
99

10+
logging.basicConfig(level=logging.INFO) # root logger defaults to WARNING; we want the INFO lines too
1011
log = logging.getLogger("demo-api")
1112
app = FastAPI()
1213

0 commit comments

Comments
 (0)