Skip to content

Commit 339b7ad

Browse files
masterkainmasterkain
committed
docs(monitoring): update runtime triage dashboard
1 parent 555c7db commit 339b7ad

1 file changed

Lines changed: 347 additions & 2 deletions

File tree

‎docs-site/public/operators/monitoring/codex-pooler-runtime-triage.json‎

Lines changed: 347 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"kind": "Dashboard",
44
"metadata": {
55
"name": "codex_pooler_runtime_triage",
6-
"generation": 5,
6+
"generation": 6,
77
"creationTimestamp": "2026-06-01T13:36:25Z",
88
"labels": {},
99
"annotations": {}
@@ -33,7 +33,7 @@
3333
}
3434
],
3535
"cursorSync": "Crosshair",
36-
"description": "Production triage view for Codex Pooler runtime health, memory pressure, gateway admission, HTTP traffic, database query pressure, and restart evidence.",
36+
"description": "Production triage view for Codex Pooler runtime health, memory pressure, gateway admission, HTTP traffic, database query pressure, admin stats dashboard reloads, and restart evidence.",
3737
"editable": true,
3838
"elements": {
3939
"panel-10": {
@@ -4192,6 +4192,351 @@
41924192
"version": "12.3.1"
41934193
}
41944194
}
4195+
},
4196+
"panel-40": {
4197+
"kind": "Panel",
4198+
"spec": {
4199+
"data": {
4200+
"kind": "QueryGroup",
4201+
"spec": {
4202+
"queries": [
4203+
{
4204+
"kind": "PanelQuery",
4205+
"spec": {
4206+
"hidden": false,
4207+
"query": {
4208+
"datasource": {
4209+
"name": "${datasource}"
4210+
},
4211+
"group": "prometheus",
4212+
"kind": "DataQuery",
4213+
"spec": {
4214+
"editorMode": "code",
4215+
"expr": "sum by (stage, window, scope) (rate(codex_pooler_admin_stats_reload_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))",
4216+
"interval": "$resolution",
4217+
"legendFormat": "{{stage}} {{window}} {{scope}}"
4218+
},
4219+
"version": "v0"
4220+
},
4221+
"refId": "A"
4222+
}
4223+
}
4224+
],
4225+
"queryOptions": {},
4226+
"transformations": []
4227+
}
4228+
},
4229+
"description": "Admin stats LiveView reload scheduling, coalescing, cancellation, and execution rate by window and scope. Coalesced spikes mean a page is already waiting on one debounced reload.",
4230+
"id": 40,
4231+
"links": [],
4232+
"title": "Admin Stats Reload Rate",
4233+
"vizConfig": {
4234+
"group": "timeseries",
4235+
"kind": "VizConfig",
4236+
"spec": {
4237+
"fieldConfig": {
4238+
"defaults": {
4239+
"color": {
4240+
"mode": "palette-classic"
4241+
},
4242+
"custom": {
4243+
"axisBorderShow": false,
4244+
"axisCenteredZero": false,
4245+
"axisColorMode": "text",
4246+
"axisLabel": "",
4247+
"axisPlacement": "auto",
4248+
"barAlignment": 0,
4249+
"barWidthFactor": 0.6,
4250+
"drawStyle": "line",
4251+
"fillOpacity": 15,
4252+
"gradientMode": "opacity",
4253+
"hideFrom": {
4254+
"legend": false,
4255+
"tooltip": false,
4256+
"viz": false
4257+
},
4258+
"insertNulls": false,
4259+
"lineInterpolation": "linear",
4260+
"lineWidth": 2,
4261+
"pointSize": 5,
4262+
"scaleDistribution": {
4263+
"type": "linear"
4264+
},
4265+
"showPoints": "never",
4266+
"showValues": false,
4267+
"spanNulls": false,
4268+
"stacking": {
4269+
"group": "A",
4270+
"mode": "none"
4271+
},
4272+
"thresholdsStyle": {
4273+
"mode": "off"
4274+
}
4275+
},
4276+
"mappings": [],
4277+
"thresholds": {
4278+
"mode": "absolute",
4279+
"steps": [
4280+
{
4281+
"color": "green",
4282+
"value": 0
4283+
}
4284+
]
4285+
},
4286+
"unit": "ops"
4287+
},
4288+
"overrides": []
4289+
},
4290+
"options": {
4291+
"legend": {
4292+
"calcs": [
4293+
"lastNotNull",
4294+
"max"
4295+
],
4296+
"displayMode": "table",
4297+
"placement": "bottom",
4298+
"showLegend": true
4299+
},
4300+
"tooltip": {
4301+
"hideZeros": false,
4302+
"mode": "multi",
4303+
"sort": "desc"
4304+
}
4305+
}
4306+
},
4307+
"version": "12.3.1"
4308+
}
4309+
}
4310+
},
4311+
"panel-41": {
4312+
"kind": "Panel",
4313+
"spec": {
4314+
"data": {
4315+
"kind": "QueryGroup",
4316+
"spec": {
4317+
"queries": [
4318+
{
4319+
"kind": "PanelQuery",
4320+
"spec": {
4321+
"hidden": false,
4322+
"query": {
4323+
"datasource": {
4324+
"name": "${datasource}"
4325+
},
4326+
"group": "prometheus",
4327+
"kind": "DataQuery",
4328+
"spec": {
4329+
"editorMode": "code",
4330+
"expr": "sum by (outcome, window, scope) (rate(codex_pooler_admin_stats_dashboard_build_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))",
4331+
"interval": "$resolution",
4332+
"legendFormat": "{{outcome}} {{window}} {{scope}}"
4333+
},
4334+
"version": "v0"
4335+
},
4336+
"refId": "A"
4337+
}
4338+
}
4339+
],
4340+
"queryOptions": {},
4341+
"transformations": []
4342+
}
4343+
},
4344+
"description": "Full admin stats dashboard builds by outcome, selected time window, and scope. This shows the reload work that can drive read-model DB pressure from open admin pages.",
4345+
"id": 41,
4346+
"links": [],
4347+
"title": "Admin Stats Dashboard Build Rate",
4348+
"vizConfig": {
4349+
"group": "timeseries",
4350+
"kind": "VizConfig",
4351+
"spec": {
4352+
"fieldConfig": {
4353+
"defaults": {
4354+
"color": {
4355+
"mode": "palette-classic"
4356+
},
4357+
"custom": {
4358+
"axisBorderShow": false,
4359+
"axisCenteredZero": false,
4360+
"axisColorMode": "text",
4361+
"axisLabel": "",
4362+
"axisPlacement": "auto",
4363+
"barAlignment": 0,
4364+
"barWidthFactor": 0.6,
4365+
"drawStyle": "line",
4366+
"fillOpacity": 15,
4367+
"gradientMode": "opacity",
4368+
"hideFrom": {
4369+
"legend": false,
4370+
"tooltip": false,
4371+
"viz": false
4372+
},
4373+
"insertNulls": false,
4374+
"lineInterpolation": "linear",
4375+
"lineWidth": 2,
4376+
"pointSize": 5,
4377+
"scaleDistribution": {
4378+
"type": "linear"
4379+
},
4380+
"showPoints": "never",
4381+
"showValues": false,
4382+
"spanNulls": false,
4383+
"stacking": {
4384+
"group": "A",
4385+
"mode": "none"
4386+
},
4387+
"thresholdsStyle": {
4388+
"mode": "off"
4389+
}
4390+
},
4391+
"mappings": [],
4392+
"thresholds": {
4393+
"mode": "absolute",
4394+
"steps": [
4395+
{
4396+
"color": "green",
4397+
"value": 0
4398+
}
4399+
]
4400+
},
4401+
"unit": "ops"
4402+
},
4403+
"overrides": []
4404+
},
4405+
"options": {
4406+
"legend": {
4407+
"calcs": [
4408+
"lastNotNull",
4409+
"max"
4410+
],
4411+
"displayMode": "table",
4412+
"placement": "bottom",
4413+
"showLegend": true
4414+
},
4415+
"tooltip": {
4416+
"hideZeros": false,
4417+
"mode": "multi",
4418+
"sort": "desc"
4419+
}
4420+
}
4421+
},
4422+
"version": "12.3.1"
4423+
}
4424+
}
4425+
},
4426+
"panel-42": {
4427+
"kind": "Panel",
4428+
"spec": {
4429+
"data": {
4430+
"kind": "QueryGroup",
4431+
"spec": {
4432+
"queries": [
4433+
{
4434+
"kind": "PanelQuery",
4435+
"spec": {
4436+
"hidden": false,
4437+
"query": {
4438+
"datasource": {
4439+
"name": "${datasource}"
4440+
},
4441+
"group": "prometheus",
4442+
"kind": "DataQuery",
4443+
"spec": {
4444+
"editorMode": "code",
4445+
"expr": "histogram_quantile(0.95, sum by (le, outcome, window, scope) (rate(codex_pooler_admin_stats_dashboard_build_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))",
4446+
"interval": "$resolution",
4447+
"legendFormat": "{{outcome}} {{window}} {{scope}} p95"
4448+
},
4449+
"version": "v0"
4450+
},
4451+
"refId": "A"
4452+
}
4453+
}
4454+
],
4455+
"queryOptions": {},
4456+
"transformations": []
4457+
}
4458+
},
4459+
"description": "p95 duration for full admin stats dashboard builds. Pair spikes here with hot query source and DB queue panels to identify expensive stats reload paths.",
4460+
"id": 42,
4461+
"links": [],
4462+
"title": "Admin Stats Dashboard Build p95",
4463+
"vizConfig": {
4464+
"group": "timeseries",
4465+
"kind": "VizConfig",
4466+
"spec": {
4467+
"fieldConfig": {
4468+
"defaults": {
4469+
"color": {
4470+
"mode": "palette-classic"
4471+
},
4472+
"custom": {
4473+
"axisBorderShow": false,
4474+
"axisCenteredZero": false,
4475+
"axisColorMode": "text",
4476+
"axisLabel": "",
4477+
"axisPlacement": "auto",
4478+
"barAlignment": 0,
4479+
"barWidthFactor": 0.6,
4480+
"drawStyle": "line",
4481+
"fillOpacity": 15,
4482+
"gradientMode": "opacity",
4483+
"hideFrom": {
4484+
"legend": false,
4485+
"tooltip": false,
4486+
"viz": false
4487+
},
4488+
"insertNulls": false,
4489+
"lineInterpolation": "linear",
4490+
"lineWidth": 2,
4491+
"pointSize": 5,
4492+
"scaleDistribution": {
4493+
"type": "linear"
4494+
},
4495+
"showPoints": "never",
4496+
"showValues": false,
4497+
"spanNulls": false,
4498+
"stacking": {
4499+
"group": "A",
4500+
"mode": "none"
4501+
},
4502+
"thresholdsStyle": {
4503+
"mode": "off"
4504+
}
4505+
},
4506+
"mappings": [],
4507+
"thresholds": {
4508+
"mode": "absolute",
4509+
"steps": [
4510+
{
4511+
"color": "green",
4512+
"value": 0
4513+
}
4514+
]
4515+
},
4516+
"unit": "s"
4517+
},
4518+
"overrides": []
4519+
},
4520+
"options": {
4521+
"legend": {
4522+
"calcs": [
4523+
"lastNotNull",
4524+
"max"
4525+
],
4526+
"displayMode": "table",
4527+
"placement": "bottom",
4528+
"showLegend": true
4529+
},
4530+
"tooltip": {
4531+
"hideZeros": false,
4532+
"mode": "multi",
4533+
"sort": "desc"
4534+
}
4535+
}
4536+
},
4537+
"version": "12.3.1"
4538+
}
4539+
}
41954540
}
41964541
},
41974542
"layout": {

0 commit comments

Comments
 (0)