diff --git a/prometheus-grafana/grafana/dashboards/trainus-dashboard.json b/prometheus-grafana/grafana/dashboards/trainus-dashboard.json index 84dc818..38786f0 100644 --- a/prometheus-grafana/grafana/dashboards/trainus-dashboard.json +++ b/prometheus-grafana/grafana/dashboards/trainus-dashboard.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -15,187 +18,898 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, + "id": 4, "links": [], "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 100, "panels": [], "title": "System Resources (Threads & Connections) [$env]", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus-default" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus-default" + }, "description": "API 서버의 Tomcat 스레드 상태 (Busy, Current, Max)", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 2 }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "short" - } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, "id": 1, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.5.2", "targets": [ - { "expr": "tomcat_threads_busy_threads{env=\"$env\", role=\"api\"}", "legendFormat": "Busy: {{instance}}", "refId": "A" }, - { "expr": "tomcat_threads_current_threads{env=\"$env\", role=\"api\"}", "legendFormat": "Current: {{instance}}", "refId": "B" }, - { "expr": "tomcat_threads_config_max_threads{env=\"$env\", role=\"api\"}", "legendFormat": "Max: {{instance}}", "refId": "C" } + { + "expr": "tomcat_threads_busy_threads{env=\"$env\", role=\"api\"}", + "legendFormat": "Busy: {{instance}}", + "refId": "A" + }, + { + "expr": "tomcat_threads_current_threads{env=\"$env\", role=\"api\"}", + "legendFormat": "Current: {{instance}}", + "refId": "B" + }, + { + "expr": "tomcat_threads_config_max_threads{env=\"$env\", role=\"api\"}", + "legendFormat": "Max: {{instance}}", + "refId": "C" + } ], "title": "API Tomcat Threads", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus-default" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus-default" + }, "description": "API 서버의 HikariCP 커넥션 풀 상태 (Active, Idle, Pending)", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "drawStyle": "line", "fillOpacity": 15, "lineWidth": 2 }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "short" - } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, "id": 3, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.5.2", "targets": [ - { "expr": "hikaricp_connections_active{env=\"$env\", role=\"api\"}", "legendFormat": "Active: {{instance}}", "refId": "A" }, - { "expr": "hikaricp_connections_idle{env=\"$env\", role=\"api\"}", "legendFormat": "Idle: {{instance}}", "refId": "B" }, - { "expr": "hikaricp_connections_pending{env=\"$env\", role=\"api\"}", "legendFormat": "Pending: {{instance}}", "refId": "C" } + { + "expr": "hikaricp_connections_active{env=\"$env\", role=\"api\"}", + "legendFormat": "Active: {{instance}}", + "refId": "A" + }, + { + "expr": "hikaricp_connections_idle{env=\"$env\", role=\"api\"}", + "legendFormat": "Idle: {{instance}}", + "refId": "B" + }, + { + "expr": "hikaricp_connections_pending{env=\"$env\", role=\"api\"}", + "legendFormat": "Pending: {{instance}}", + "refId": "C" + } ], "title": "API HikariCP Connections", "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 101, "panels": [], "title": "Core Business Metrics (MQ & Defense) [$env]", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus-default" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus-default" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 2 }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "s" - } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 10 }, - "gridPos": { "h": 8, "w": 16, "x": 0, "y": 10 }, "id": 120, "options": { - "legend": { "calcs": ["max", "lastNotNull"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, + "pluginVersion": "11.5.2", "targets": [ - { "expr": "max(lesson_apply_latency_seconds_max{env=\"$env\"}) by (instance, instance_id)", "legendFormat": "Latency: {{instance}}", "refId": "A" } + { + "expr": "max(lesson_apply_latency_seconds_max{env=\"$env\"}) by (instance, instance_id)", + "legendFormat": "Latency: {{instance}}", + "refId": "A" + } ], "title": "Async Processing Latency (Max)", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus-default" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus-default" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } ] - } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 122, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "editorMode": "code", + "expr": " sum by (instance, instance_id) (\n rate(lesson_apply_latency_seconds_sum{env=\"$env\"}[$__rate_interval])\n )\n /\n sum by (instance, instance_id) (\n rate(lesson_apply_latency_seconds_count{env=\"$env\"}[$__rate_interval])\n )", + "legendFormat": "Latency: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Async Processing Latency (average))", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus-default" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 123, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(\n rate(lesson_apply_latency_seconds_count{env=\"$env\"}[$__rate_interval])\n)", + "legendFormat": "Latency: {{instance}}", + "range": true, + "refId": "A" } + ], + "title": "Async Processing Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus-default" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 10 }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 10 }, "id": 121, "options": { - "legend": { "calcs": ["max", "lastNotNull"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, + "pluginVersion": "11.5.2", "targets": [ - { "expr": "sum(max_over_time(lesson_apply_total{env=\"$env\", result=\"pre_filter_reject\"}[$__range]) - min_over_time(lesson_apply_total{env=\"$env\", result=\"pre_filter_reject\"}[$__range]))", "legendFormat": "Rejected (Exact Count)", "refId": "A" } + { + "expr": "sum(max_over_time(lesson_apply_total{env=\"$env\", result=\"pre_filter_reject\"}[$__range]) - min_over_time(lesson_apply_total{env=\"$env\", result=\"pre_filter_reject\"}[$__range]))", + "legendFormat": "Rejected (Exact Count)", + "refId": "A" + } ], "title": "[Defend] Pre-filter Reject (Real-time Steps)", "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, "id": 102, "panels": [], "title": "Throughput & Latency (HTTP) [$env]", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus-default" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus-default" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "drawStyle": "line", "lineWidth": 2 }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "reqps" - } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", "targets": [ - { "expr": "sum(rate(http_server_requests_seconds_count{env=\"$env\"}[1m])) by (uri, instance)", "legendFormat": "{{uri}} ({{instance}})", "refId": "A" } + { + "expr": "sum(rate(http_server_requests_seconds_count{env=\"$env\"}[1m])) by (uri, instance)", + "legendFormat": "{{uri}} ({{instance}})", + "refId": "A" + } ], "title": "Request Throughput (RPS)", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus-default" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus-default" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "drawStyle": "line", "lineWidth": 2 }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "s" - } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "/health (172.31.3.198:8082)" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", "targets": [ - { "expr": "rate(http_server_requests_seconds_sum{env=\"$env\"}[1m]) / rate(http_server_requests_seconds_count{env=\"$env\"}[1m])", "legendFormat": "{{uri}} ({{instance}})", "refId": "A" } + { + "expr": "rate(http_server_requests_seconds_sum{env=\"$env\"}[1m]) / rate(http_server_requests_seconds_count{env=\"$env\"}[1m])", + "legendFormat": "{{uri}} ({{instance}})", + "refId": "A" + } ], "title": "Average Response Time", "type": "timeseries" } ], + "preload": false, "refresh": "5s", "schemaVersion": 40, + "tags": [], "templating": { "list": [ { - "current": { "selected": true, "text": "local", "value": "local" }, - "hide": 0, + "current": { + "text": "production", + "value": "production" + }, "includeAll": false, "label": "Environment", - "multi": false, "name": "env", "options": [ - { "selected": true, "text": "local", "value": "local" }, - { "selected": false, "text": "production", "value": "production" }, - { "selected": false, "text": "baseline", "value": "baseline" } + { + "selected": false, + "text": "local", + "value": "local" + }, + { + "selected": true, + "text": "production", + "value": "production" + }, + { + "selected": false, + "text": "baseline", + "value": "baseline" + } ], "query": "local,production,baseline", "type": "custom" } ] }, - "time": { "from": "now-15m", "to": "now" }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", "title": "TrainUs Master Dashboard", "uid": "trainus-master-v1", - "version": 4 + "version": 12, + "weekStart": "" }