|
| 1 | +--- |
| 2 | +# Cross-stack correlation dashboard. Each row puts two related signals on the same |
| 3 | +# time axis so you can eyeball causation: restart-vs-nodes, softirq-vs-retrans, etc. |
| 4 | +apiVersion: v1 |
| 5 | +kind: ConfigMap |
| 6 | +metadata: |
| 7 | + name: pilot-gcp-correlation-dashboard |
| 8 | + namespace: monitoring |
| 9 | + labels: {grafana_dashboard: "1"} |
| 10 | + annotations: {grafana_folder: "Pilot GCP"} |
| 11 | +data: |
| 12 | + pilot-gcp-correlation.json: | |
| 13 | + { |
| 14 | + "title": "Pilot GCP — Correlation (how things tie together)", |
| 15 | + "uid": "pilot-gcp-correlation", |
| 16 | + "schemaVersion": 38, |
| 17 | + "timezone": "", |
| 18 | + "refresh": "30s", |
| 19 | + "time": {"from": "now-6h", "to": "now"}, |
| 20 | + "tags": ["pilot", "gcp", "correlation"], |
| 21 | + "links": [ |
| 22 | + {"title": "Fleet", "url": "/d/pilot-gcp-fleet", "type": "link"}, |
| 23 | + {"title": "Diagnostics", "url": "/d/pilot-gcp-diagnostics", "type": "link"}, |
| 24 | + {"title": "VM Explorer", "url": "/d/pilot-gcp-vm-explorer", "type": "link"} |
| 25 | + ], |
| 26 | + "panels": [ |
| 27 | + {"id": 1, "type": "row", "title": "Rendezvous — control-plane cascade", "collapsed": false, |
| 28 | + "gridPos": {"x": 0, "y": 0, "w": 24, "h": 1}}, |
| 29 | +
|
| 30 | + {"id": 10, "type": "timeseries", "title": "Online nodes + restart events", |
| 31 | + "description": "Drops in online_nodes that align with restart_events = the restart caused the drop.", |
| 32 | + "gridPos": {"x": 0, "y": 1, "w": 12, "h": 8}, |
| 33 | + "targets": [ |
| 34 | + {"expr": "pilot_online_nodes", "refId": "A", "legendFormat": "online"}, |
| 35 | + {"expr": "pilot_total_nodes", "refId": "B", "legendFormat": "total"}, |
| 36 | + {"expr": "pilot_rendezvous_restart_events", "refId": "C", "legendFormat": "restart count (cumul)"} |
| 37 | + ]}, |
| 38 | +
|
| 39 | + {"id": 11, "type": "timeseries", "title": "Rendezvous CPU softirq vs request rate", |
| 40 | + "description": "Softirq time tracks packet processing. If softirq saturates but requests don't rise, we're bottlenecked on kernel networking.", |
| 41 | + "gridPos": {"x": 12, "y": 1, "w": 12, "h": 8}, |
| 42 | + "targets": [ |
| 43 | + {"expr": "sum(rate(node_cpu_seconds_total{vm=\"pilot-rendezvous\",mode=\"softirq\"}[5m]))", "refId": "A", "legendFormat": "softirq cores"}, |
| 44 | + {"expr": "rate(pilot_total_requests[5m])", "refId": "B", "legendFormat": "req/s"} |
| 45 | + ]}, |
| 46 | +
|
| 47 | + {"id": 12, "type": "timeseries", "title": "TCP ESTABLISHED vs conntrack entries (rendezvous)", |
| 48 | + "description": "Rendezvous carries ~20k TCP. If conntrack climbs past connections, NAT/firewall churn is at play.", |
| 49 | + "gridPos": {"x": 0, "y": 9, "w": 12, "h": 8}, |
| 50 | + "targets": [ |
| 51 | + {"expr": "node_netstat_Tcp_CurrEstab{vm=\"pilot-rendezvous\"}", "refId": "A", "legendFormat": "TCP established"}, |
| 52 | + {"expr": "node_nf_conntrack_entries{vm=\"pilot-rendezvous\"}", "refId": "B", "legendFormat": "conntrack entries"}, |
| 53 | + {"expr": "node_filefd_allocated{vm=\"pilot-rendezvous\"}", "refId": "C", "legendFormat": "file descriptors"} |
| 54 | + ]}, |
| 55 | +
|
| 56 | + {"id": 13, "type": "timeseries", "title": "Rendezvous retrans + probe staleness", |
| 57 | + "description": "If retrans climbs and probe_staleness climbs together = network to rendezvous is degrading.", |
| 58 | + "gridPos": {"x": 12, "y": 9, "w": 12, "h": 8}, |
| 59 | + "targets": [ |
| 60 | + {"expr": "rate(node_netstat_Tcp_RetransSegs{vm=\"pilot-rendezvous\"}[5m])", "refId": "A", "legendFormat": "rendezvous retrans /s"}, |
| 61 | + {"expr": "max by (probe) (pilot_probe_staleness_seconds)", "refId": "B", "legendFormat": "probe {{probe}} staleness"} |
| 62 | + ]}, |
| 63 | +
|
| 64 | + {"id": 20, "type": "row", "title": "Beacons — load-balancer symmetry", "collapsed": false, |
| 65 | + "gridPos": {"x": 0, "y": 17, "w": 24, "h": 1}}, |
| 66 | +
|
| 67 | + {"id": 21, "type": "timeseries", "title": "UDP pps per beacon (should overlap)", |
| 68 | + "description": "Ideal: all 6 lines track together. Divergence = LB bias.", |
| 69 | + "gridPos": {"x": 0, "y": 18, "w": 12, "h": 8}, |
| 70 | + "targets": [{"expr": "rate(node_netstat_Udp_InDatagrams{role=\"beacon\"}[5m])", "refId": "A", "legendFormat": "{{vm}}"}], |
| 71 | + "fieldConfig": {"defaults": {"unit": "pps"}}}, |
| 72 | +
|
| 73 | + {"id": 22, "type": "timeseries", "title": "Network TX per beacon", |
| 74 | + "gridPos": {"x": 12, "y": 18, "w": 12, "h": 8}, |
| 75 | + "targets": [{"expr": "rate(node_network_transmit_bytes_total{role=\"beacon\",device=\"ens4\"}[5m])", "refId": "A", "legendFormat": "{{vm}}"}], |
| 76 | + "fieldConfig": {"defaults": {"unit": "Bps"}}}, |
| 77 | +
|
| 78 | + {"id": 23, "type": "bargauge", "title": "Beacon TX share now (even = 16.7%)", |
| 79 | + "description": "Normalized share of total beacon TX. Uniform bars = symmetric LB.", |
| 80 | + "gridPos": {"x": 0, "y": 26, "w": 12, "h": 6}, |
| 81 | + "targets": [{"expr": "sum by (vm) (rate(node_network_transmit_bytes_total{role=\"beacon\",device=\"ens4\"}[5m])) / scalar(sum(rate(node_network_transmit_bytes_total{role=\"beacon\",device=\"ens4\"}[5m])))", "refId": "A", "legendFormat": "{{vm}}"}], |
| 82 | + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 0.5, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 0.3}, {"color": "red", "value": 0.4}]}}}, |
| 83 | + "options": {"displayMode": "basic", "orientation": "horizontal"}}, |
| 84 | +
|
| 85 | + {"id": 24, "type": "stat", "title": "Beacon LB imbalance ratio (max/avg)", |
| 86 | + "description": "1.0 = perfect balance. >2 = one beacon taking 2x its share.", |
| 87 | + "gridPos": {"x": 12, "y": 26, "w": 6, "h": 6}, |
| 88 | + "targets": [{"expr": "max(rate(node_network_transmit_bytes_total{role=\"beacon\",device=\"ens4\"}[5m])) / avg(rate(node_network_transmit_bytes_total{role=\"beacon\",device=\"ens4\"}[5m]))", "refId": "A"}], |
| 89 | + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background"}, |
| 90 | + "fieldConfig": {"defaults": {"unit": "short", "decimals": 2, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1.5}, {"color": "red", "value": 3}]}}}}, |
| 91 | +
|
| 92 | + {"id": 25, "type": "stat", "title": "Beacon pps stddev", |
| 93 | + "description": "σ across beacons. Low = balanced.", |
| 94 | + "gridPos": {"x": 18, "y": 26, "w": 6, "h": 6}, |
| 95 | + "targets": [{"expr": "stddev(rate(node_netstat_Udp_InDatagrams{role=\"beacon\"}[5m]))", "refId": "A"}], |
| 96 | + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}}, |
| 97 | + "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}}}, |
| 98 | +
|
| 99 | + {"id": 30, "type": "row", "title": "Agents — RTT vs reachability vs CPU", "collapsed": false, |
| 100 | + "gridPos": {"x": 0, "y": 32, "w": 24, "h": 1}}, |
| 101 | +
|
| 102 | + {"id": 31, "type": "timeseries", "title": "Probe RTT vs agent CPU", |
| 103 | + "description": "If RTT climbs when probe's agent CPU saturates, agent is the bottleneck, not the network.", |
| 104 | + "gridPos": {"x": 0, "y": 33, "w": 12, "h": 8}, |
| 105 | + "targets": [ |
| 106 | + {"expr": "pilot_probe_rtt_ms", "refId": "A", "legendFormat": "RTT {{agent}} ms"}, |
| 107 | + {"expr": "100 - avg by (vm) (rate(node_cpu_seconds_total{vm=~\".*agent.*\",mode=\"idle\"}[5m])) * 100", "refId": "B", "legendFormat": "{{vm}} cpu%"} |
| 108 | + ]}, |
| 109 | +
|
| 110 | + {"id": 32, "type": "timeseries", "title": "Probe reachability by cmd", |
| 111 | + "description": "Per-command reachability to the probe's agent. Drops = specific cmd broken.", |
| 112 | + "gridPos": {"x": 12, "y": 33, "w": 12, "h": 8}, |
| 113 | + "targets": [{"expr": "pilot_probe_reachable", "refId": "A", "legendFormat": "{{agent}} / {{cmd}}"}], |
| 114 | + "fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 1}}}, |
| 115 | +
|
| 116 | + {"id": 33, "type": "timeseries", "title": "NAT agents — TCP retrans (path health signal)", |
| 117 | + "description": "All 3 NAT agents have high baseline retrans (~800/s). Spikes above baseline = path degradation to peers.", |
| 118 | + "gridPos": {"x": 0, "y": 41, "w": 12, "h": 7}, |
| 119 | + "targets": [{"expr": "rate(node_netstat_Tcp_RetransSegs{role=\"nat-agent\"}[5m])", "refId": "A", "legendFormat": "{{vm}}"}]}, |
| 120 | +
|
| 121 | + {"id": 34, "type": "timeseries", "title": "NAT agents — TIME_WAIT (conn churn)", |
| 122 | + "description": "NAT agents churn ~2500 TW each. Climbing unbounded = outbound connections not closing cleanly.", |
| 123 | + "gridPos": {"x": 12, "y": 41, "w": 12, "h": 7}, |
| 124 | + "targets": [{"expr": "node_sockstat_TCP_tw{role=\"nat-agent\"}", "refId": "A", "legendFormat": "{{vm}}"}]}, |
| 125 | +
|
| 126 | + {"id": 40, "type": "row", "title": "Swarm — identity spoof footprint", "collapsed": false, |
| 127 | + "gridPos": {"x": 0, "y": 48, "w": 24, "h": 1}}, |
| 128 | +
|
| 129 | + {"id": 41, "type": "timeseries", "title": "UDP sockets + file descriptors (swarm-1)", |
| 130 | + "description": "Each spoofed identity opens a UDP socket + file descriptors. Fleet size proxy.", |
| 131 | + "gridPos": {"x": 0, "y": 49, "w": 12, "h": 7}, |
| 132 | + "targets": [ |
| 133 | + {"expr": "node_sockstat_UDP_inuse{vm=~\"pilot-swarm-.*\"}", "refId": "A", "legendFormat": "{{vm}} UDP sockets"}, |
| 134 | + {"expr": "node_filefd_allocated{vm=~\"pilot-swarm-.*\"}", "refId": "B", "legendFormat": "{{vm}} fds"} |
| 135 | + ]}, |
| 136 | +
|
| 137 | + {"id": 42, "type": "timeseries", "title": "Context switches + TCP established (swarm)", |
| 138 | + "description": "High context switches = traffic-generating goroutines active. TCP establish = connections they have open.", |
| 139 | + "gridPos": {"x": 12, "y": 49, "w": 12, "h": 7}, |
| 140 | + "targets": [ |
| 141 | + {"expr": "rate(node_context_switches_total{vm=~\"pilot-swarm-.*\"}[5m])", "refId": "A", "legendFormat": "{{vm}} ctxsw /s"}, |
| 142 | + {"expr": "node_netstat_Tcp_CurrEstab{vm=~\"pilot-swarm-.*\"}", "refId": "B", "legendFormat": "{{vm}} TCP est"} |
| 143 | + ]}, |
| 144 | +
|
| 145 | + {"id": 50, "type": "row", "title": "Probe — agent path health over time", "collapsed": false, |
| 146 | + "gridPos": {"x": 0, "y": 56, "w": 24, "h": 1}}, |
| 147 | +
|
| 148 | + {"id": 51, "type": "timeseries", "title": "Bench throughput vs RTT", |
| 149 | + "description": "Throughput should scale inversely with RTT (BDP). If both bad = congested link; just RTT bad = queue delay.", |
| 150 | + "gridPos": {"x": 0, "y": 57, "w": 12, "h": 7}, |
| 151 | + "targets": [ |
| 152 | + {"expr": "pilot_probe_bench_mbps * 100", "refId": "A", "legendFormat": "{{agent}} mbps×100"}, |
| 153 | + {"expr": "pilot_probe_rtt_ms", "refId": "B", "legendFormat": "{{agent}} RTT ms"} |
| 154 | + ]}, |
| 155 | +
|
| 156 | + {"id": 52, "type": "timeseries", "title": "Probe iteration + error rate", |
| 157 | + "description": "Iterations /s = probe cadence. Errors /s = cmd failures. Ratio = error %.", |
| 158 | + "gridPos": {"x": 12, "y": 57, "w": 12, "h": 7}, |
| 159 | + "targets": [ |
| 160 | + {"expr": "rate(pilot_probe_iterations_total[5m])", "refId": "A", "legendFormat": "iterations /s"}, |
| 161 | + {"expr": "rate(pilot_probe_errors_total[5m])", "refId": "B", "legendFormat": "errors /s"} |
| 162 | + ]}, |
| 163 | +
|
| 164 | + {"id": 53, "type": "timeseries", "title": "Ping loss ratio + cmd duration", |
| 165 | + "description": "Packet loss with normal duration = lossy wifi. Loss with long duration = stalling.", |
| 166 | + "gridPos": {"x": 0, "y": 64, "w": 12, "h": 7}, |
| 167 | + "targets": [ |
| 168 | + {"expr": "pilot_probe_ping_loss_ratio", "refId": "A", "legendFormat": "{{agent}} loss"}, |
| 169 | + {"expr": "pilot_probe_cmd_duration_seconds", "refId": "B", "legendFormat": "{{agent}}/{{cmd}} dur"} |
| 170 | + ]}, |
| 171 | +
|
| 172 | + {"id": 54, "type": "timeseries", "title": "Daemon version age (auto-updater health)", |
| 173 | + "description": "Age since probe daemon was built. Climbing without resets = auto-updater not updating.", |
| 174 | + "gridPos": {"x": 12, "y": 64, "w": 12, "h": 7}, |
| 175 | + "targets": [ |
| 176 | + {"expr": "pilot_probe_version_age_seconds", "refId": "A", "legendFormat": "age (s)"}, |
| 177 | + {"expr": "pilot_probe_version_matches_github", "refId": "B", "legendFormat": "matches github"} |
| 178 | + ]}, |
| 179 | +
|
| 180 | + {"id": 60, "type": "row", "title": "Scriptorium — disk/inode vs activity", "collapsed": false, |
| 181 | + "gridPos": {"x": 0, "y": 71, "w": 24, "h": 1}}, |
| 182 | +
|
| 183 | + {"id": 61, "type": "timeseries", "title": "Inodes free + disk write rate (scriptorium)", |
| 184 | + "description": "Dropping inodes with sustained write rate = inbox accumulation. Cron at 00:00 should recover.", |
| 185 | + "gridPos": {"x": 0, "y": 72, "w": 12, "h": 7}, |
| 186 | + "targets": [ |
| 187 | + {"expr": "node_filesystem_files_free{vm=\"scriptorium-server\",mountpoint=\"/\"}", "refId": "A", "legendFormat": "inodes free"}, |
| 188 | + {"expr": "sum(rate(node_disk_written_bytes_total{vm=\"scriptorium-server\"}[5m]))", "refId": "B", "legendFormat": "write B/s"} |
| 189 | + ]}, |
| 190 | +
|
| 191 | + {"id": 62, "type": "timeseries", "title": "Scriptorium TCP + UDP traffic", |
| 192 | + "description": "Correlate traffic spikes with inbox growth.", |
| 193 | + "gridPos": {"x": 12, "y": 72, "w": 12, "h": 7}, |
| 194 | + "targets": [ |
| 195 | + {"expr": "node_netstat_Tcp_CurrEstab{vm=\"scriptorium-server\"}", "refId": "A", "legendFormat": "TCP est"}, |
| 196 | + {"expr": "rate(node_netstat_Udp_InDatagrams{vm=\"scriptorium-server\"}[5m])", "refId": "B", "legendFormat": "UDP in /s"} |
| 197 | + ]} |
| 198 | + ] |
| 199 | + } |
0 commit comments