Skip to content

Commit 5d3ca75

Browse files
mrsillydogmrsillydog
authored
PromQL Alerts: Google GKE (#1182)
* MQL * PromQL * PR feedback * Revert "PR feedback" This reverts commit b66ebd2. * Query fixes * Restore userLabels * Update documentation * = => =~ * Reversions and extra doc vars
1 parent fd1ca6a commit 5d3ca75

19 files changed

Lines changed: 147 additions & 104 deletions

‎alerts/google-gke/cpu-limit-utilization-entire-workload.v1.json‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,29 @@
11
{
22
"displayName": "High CPU Limit Utilization (GKE Workload: ${WORKLOAD_NAME})",
33
"documentation": {
4-
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${CLUSTER_LOCATION}/${CLUSTER_NAME}/${NAMESPACE}/${WORKLOAD_NAME}/observability?project=${PROJECT_ID}).\n- Workloads with high CPU limit utilization likely have containers that are CPU throttled. To avoid application slowdown and unresponsiveness, you can either keep CPU usage below the CPU limit or increase the limit [View Documentation](https://cloud.google.com/blog/products/containers-kubernetes/kubernetes-best-practices-resource-requests-and-limits)\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [MQL documentation](https://cloud.google.com/monitoring/mql/alerts)",
4+
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${metric_or_resource.label.location}/${metric_or_resource.label.cluster_name}/${metric_or_resource.label.namespace_name}/${metric.label.metadata_system_top_level_controller_name}/observability?project=${metric_or_resource.label.project_id}).\n- Workloads with high CPU limit utilization likely have containers that are CPU throttled. To avoid application slowdown and unresponsiveness, you can either keep CPU usage below the CPU limit or increase the limit [View Documentation](https://cloud.google.com/blog/products/containers-kubernetes/kubernetes-best-practices-resource-requests-and-limits)\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [PromQL documentation](https://cloud.google.com/monitoring/promql/create-promql-alerts-console)",
55
"mimeType": "text/markdown"
66
},
7+
"userLabels": {},
78
"conditions": [
89
{
910
"displayName": "CPU Limit Utilization is high for: ${WORKLOAD_NAME}",
10-
"conditionMonitoringQueryLanguage": {
11+
"conditionPrometheusQueryLanguage": {
1112
"duration": "0s",
12-
"trigger": {
13-
"count": 1
14-
},
15-
"query": "fetch k8s_container\n| \n{\n metric 'kubernetes.io/container/cpu/core_usage_time'\n | filter\n resource.project_id == '${PROJECT_ID}'\n &&\n (metadata.system_labels.top_level_controller_name == '${WORKLOAD_NAME}'\n && metadata.system_labels.top_level_controller_type == '${TOP_LEVEL_CONTROLLER_TYPE}')\n &&\n (resource.cluster_name == '${CLUSTER_NAME}'\n && resource.location == '${CLUSTER_LOCATION}'\n && resource.namespace_name == '${NAMESPACE}')\n | align rate(5m)\n | every 1m\n | group_by [],\n [value_core_usage_time_aggregate: aggregate(value.core_usage_time)]\n;\n metric 'kubernetes.io/container/cpu/limit_cores'\n | filter\n resource.project_id == '${PROJECT_ID}'\n &&\n (metadata.system_labels.top_level_controller_name == '${WORKLOAD_NAME}'\n && metadata.system_labels.top_level_controller_type == '${TOP_LEVEL_CONTROLLER_TYPE}')\n &&\n (resource.cluster_name == '${CLUSTER_NAME}'\n && resource.location == '${CLUSTER_LOCATION}'\n && resource.namespace_name == '${NAMESPACE}')\n | group_by 5m, [value_limit_cores_mean: mean(value.limit_cores)]\n | every 1m\n | group_by [],\n [value_limit_cores_mean_aggregate: aggregate(value_limit_cores_mean)]\n}\n| join\n| div\n| map add[p: 'CPU Limit Utilization']\n| scale('%')\n| condition val() > 90'%'"
13+
"evaluationInterval": "30s",
14+
"query": "label_replace(\n sum(rate(\n {\"kubernetes.io/container/cpu/core_usage_time\",\n monitored_resource=\"k8s_container\",\n metadata_system_top_level_controller_name=~\"${WORKLOAD_NAME}\",\n metadata_system_top_level_controller_type=~\"${TOP_LEVEL_CONTROLLER_TYPE}\",\n project_id=~\"${PROJECT_ID}\",\n cluster_name=~\"${CLUSTER_NAME}\",\n location=~\"${CLUSTER_LOCATION}\",\n namespace_name=~\"${NAMESPACE}\"}[5m])\n )\n /\n sum(avg_over_time(\n {\"kubernetes.io/container/cpu/limit_cores\",\n monitored_resource=\"k8s_container\",\n metadata_system_top_level_controller_name=~\"${WORKLOAD_NAME}\",\n metadata_system_top_level_controller_type=~\"${TOP_LEVEL_CONTROLLER_TYPE}\",\n project_id=~\"${PROJECT_ID}\",\n cluster_name=~\"${CLUSTER_NAME}\",\n location=~\"${CLUSTER_LOCATION}\",\n namespace_name=~\"${NAMESPACE}\"}[5m])\n ), \"p\", \"CPU Limit Utilization\", \"\", \"\"\n)\n> 0.9"
1615
}
1716
}
1817
],
1918
"alertStrategy": {
20-
"autoClose": "604800s"
19+
"autoClose": "604800s",
20+
"notificationPrompts": [
21+
"OPENED",
22+
"CLOSED"
23+
]
2124
},
2225
"combiner": "OR",
23-
"enabled": true
26+
"enabled": true,
27+
"notificationChannels": [],
28+
"severity": "SEVERITY_UNSPECIFIED"
2429
}

‎alerts/google-gke/cpu-request-utilization-entire-workload.v1.json‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,29 @@
11
{
22
"displayName": "High CPU Request Utilization (GKE Workload: ${WORKLOAD_NAME})",
33
"documentation": {
4-
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${CLUSTER_LOCATION}/${CLUSTER_NAME}/${NAMESPACE}/${WORKLOAD_NAME}/observability?project=${PROJECT_ID}).\n- Workloads with high CPU request utilization likely have containers that are CPU throttled. To avoid application slowdown and unresponsiveness, you can either keep CPU usage below the CPU request limit or increase the limit [View Documentation](https://cloud.google.com/blog/products/containers-kubernetes/kubernetes-best-practices-resource-requests-and-limits)\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [MQL documentation](https://cloud.google.com/monitoring/mql/alerts)",
4+
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${metric_or_resource.label.location}/${metric_or_resource.label.cluster_name}/${metric_or_resource.label.namespace_name}/${metric.label.metadata_system_top_level_controller_name}/observability?project=${metric_or_resource.label.project_id}).\n- Workloads with high CPU request utilization likely have containers that are CPU throttled. To avoid application slowdown and unresponsiveness, you can either keep CPU usage below the CPU request limit or increase the limit [View Documentation](https://cloud.google.com/blog/products/containers-kubernetes/kubernetes-best-practices-resource-requests-and-limits)\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [PromQL documentation](https://cloud.google.com/monitoring/promql/create-promql-alerts-console)",
55
"mimeType": "text/markdown"
66
},
7+
"userLabels": {},
78
"conditions": [
89
{
910
"displayName": "CPU Request Utilization is high for: ${WORKLOAD_NAME}",
10-
"conditionMonitoringQueryLanguage": {
11+
"conditionPrometheusQueryLanguage": {
1112
"duration": "0s",
12-
"trigger": {
13-
"count": 1
14-
},
15-
"query": "fetch k8s_container\n| \n{\n metric 'kubernetes.io/container/cpu/core_usage_time'\n | filter\n resource.project_id == '${PROJECT_ID}'\n &&\n (metadata.system_labels.top_level_controller_name == '${WORKLOAD_NAME}'\n && metadata.system_labels.top_level_controller_type == '${TOP_LEVEL_CONTROLLER_TYPE}')\n &&\n (resource.cluster_name == '${CLUSTER_NAME}'\n && resource.location == '${CLUSTER_LOCATION}'\n && resource.namespace_name == '${NAMESPACE}')\n | align rate(5m)\n | every 1m\n | group_by [],\n [value_core_usage_time_aggregate: aggregate(value.core_usage_time)]\n;\n metric 'kubernetes.io/container/cpu/request_cores'\n | filter\n resource.project_id == '${PROJECT_ID}'\n &&\n (metadata.system_labels.top_level_controller_name == '${WORKLOAD_NAME}'\n && metadata.system_labels.top_level_controller_type == '${TOP_LEVEL_CONTROLLER_TYPE}')\n &&\n (resource.cluster_name == '${CLUSTER_NAME}'\n && resource.location == '${CLUSTER_LOCATION}'\n && resource.namespace_name == '${NAMESPACE}')\n | group_by 5m, [value_request_cores_mean: mean(value.request_cores)]\n | every 1m\n | group_by [],\n [value_request_cores_mean_aggregate: aggregate(value_request_cores_mean)]\n}\n| join\n| div\n| map add[p: 'CPU Request Utilization']\n| scale('%')\n| condition val() > 90'%'"
13+
"evaluationInterval": "30s",
14+
"query": "label_replace(\n sum(rate(\n {\"kubernetes.io/container/cpu/core_usage_time\",\n monitored_resource=\"k8s_container\",\n metadata_system_top_level_controller_name=~\"${WORKLOAD_NAME}\",\n metadata_system_top_level_controller_type=~\"${TOP_LEVEL_CONTROLLER_TYPE}\",\n project_id=~\"${PROJECT_ID}\",\n cluster_name=~\"${CLUSTER_NAME}\",\n location=~\"${CLUSTER_LOCATION}\",\n namespace_name=~\"${NAMESPACE}\"}[5m])\n )\n /\n sum(avg_over_time(\n {\"kubernetes.io/container/cpu/request_cores\",\n monitored_resource=\"k8s_container\",\n metadata_system_top_level_controller_name=~\"${WORKLOAD_NAME}\",\n metadata_system_top_level_controller_type=~\"${TOP_LEVEL_CONTROLLER_TYPE}\",\n project_id=~\"${PROJECT_ID}\",\n cluster_name=~\"${CLUSTER_NAME}\",\n location=~\"${CLUSTER_LOCATION}\",\n namespace_name=~\"${NAMESPACE}\"}[5m])\n ), \"p\", \"CPU Request Utilization\", \"\", \"\"\n)\n> 0.9"
1615
}
1716
}
1817
],
1918
"alertStrategy": {
20-
"autoClose": "604800s"
19+
"autoClose": "604800s",
20+
"notificationPrompts": [
21+
"OPENED",
22+
"CLOSED"
23+
]
2124
},
2225
"combiner": "OR",
23-
"enabled": true
26+
"enabled": true,
27+
"notificationChannels": [],
28+
"severity": "SEVERITY_UNSPECIFIED"
2429
}

‎alerts/google-gke/gke-ingress-latency-entire-workload.v1.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"displayName": "High Latency (GKE Workload: ${WORKLOAD_NAME})",
33
"documentation": {
4-
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${CLUSTER_LOCATION}/${CLUSTER_NAME}/${NAMESPACE}/${WORKLOAD_NAME}/observability?project=${PROJECT_ID}).\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [MQL documentation](https://cloud.google.com/monitoring/mql/alerts)",
4+
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${metric_or_resource.label.location}/${metric_or_resource.label.cluster_name}/${metric_or_resource.label.namespace_name}/${metric.label.metadata_system_top_level_controller_name}/observability?project=${metric_or_resource.label.project_id}).\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [PromQL documentation](https://cloud.google.com/monitoring/promql/create-promql-alerts-console)",
55
"mimeType": "text/markdown"
66
},
77
"userLabels": {

‎alerts/google-gke/gke-ingress-server-errors-entire-workload.v1.json‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"displayName": "Server Error (5xx) is high. (GKE Workload: ${WORKLOAD_NAME})",
33
"documentation": {
4-
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${CLUSTER_LOCATION}/${CLUSTER_NAME}/${NAMESPACE}/${WORKLOAD_NAME}/observability?project=${PROJECT_ID}).\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [MQL documentation](https://cloud.google.com/monitoring/mql/alerts)",
4+
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${metric_or_resource.label.location}/${metric_or_resource.label.cluster_name}/${metric_or_resource.label.namespace_name}/${metric.label.metadata_system_top_level_controller_name}/observability?project=${metric_or_resource.label.project_id}).\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [PromQL documentation](https://cloud.google.com/monitoring/promql/create-promql-alerts-console)",
55
"mimeType": "text/markdown"
66
},
77
"userLabels": {
@@ -15,19 +15,22 @@
1515
"conditions": [
1616
{
1717
"displayName": "Server Error (5xx) is high for a URL path",
18-
"conditionMonitoringQueryLanguage": {
18+
"conditionPrometheusQueryLanguage": {
1919
"duration": "0s",
20-
"trigger": {
21-
"count": 1
22-
},
23-
"query": "fetch https_lb_rule\n| metric 'loadbalancing.googleapis.com/https/request_count'\n| filter\n resource.project_id == '${PROJECT_ID}' &&\n resource.backend_name =~ '${NETWORK_ENDPOINT_GROUPS_REGEX}' &&\n resource.backend_type == 'NETWORK_ENDPOINT_GROUP'\n|\n{\n filter metric.response_code_class == 500\n | align rate(5m)\n | every 1m\n | group_by [resource.matched_url_path_rule], [value_request_count_aggregate: aggregate(value.request_count)]\n ;\n align rate(5m)\n | every 1m\n | group_by [resource.matched_url_path_rule], [value_request_count_aggregate: aggregate(value.request_count)]\n}\n| join\n| div\n| scale('%')\n| condition val() > 5'%'"
20+
"evaluationInterval": "30s",
21+
"query": "(\n sum by (matched_url_path_rule) (\n rate({\n \"loadbalancing.googleapis.com/https/request_count\",\n monitored_resource=\"https_lb_rule\",\n project_id=~\"${PROJECT_ID}\",\n backend_name=~\"${NETWORK_ENDPOINT_GROUPS_REGEX}\",\n backend_type=\"NETWORK_ENDPOINT_GROUP\",\n response_code_class=\"500\"\n }[5m])\n )\n /\n sum by (matched_url_path_rule) (\n rate({\n \"loadbalancing.googleapis.com/https/request_count\",\n monitored_resource=\"https_lb_rule\",\n project_id=~\"${PROJECT_ID}\",\n backend_name=~\"${NETWORK_ENDPOINT_GROUPS_REGEX}\",\n backend_type=\"NETWORK_ENDPOINT_GROUP\"\n }[5m])\n )\n) > 0.05"
2422
}
2523
}
2624
],
2725
"alertStrategy": {
28-
"autoClose": "604800s"
26+
"autoClose": "604800s",
27+
"notificationPrompts": [
28+
"OPENED",
29+
"CLOSED"
30+
]
2931
},
3032
"combiner": "OR",
3133
"enabled": true,
32-
"notificationChannels": []
34+
"notificationChannels": [],
35+
"severity": "SEVERITY_UNSPECIFIED"
3336
}

‎alerts/google-gke/grpc-latency-entire-workload.v1.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"displayName": "High Latency (GKE Workload: ${WORKLOAD_NAME})",
33
"documentation": {
4-
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${CLUSTER_LOCATION}/${CLUSTER_NAME}/${NAMESPACE}/${WORKLOAD_NAME}/observability?project=${PROJECT_ID}).\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [MQL documentation](https://cloud.google.com/monitoring/mql/alerts)",
4+
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${metric_or_resource.label.location}/${metric_or_resource.label.cluster_name}/${metric_or_resource.label.namespace_name}/${metric.label.metadata_system_top_level_controller_name}/observability?project=${metric_or_resource.label.project_id}).\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [PromQL documentation](https://cloud.google.com/monitoring/promql/create-promql-alerts-console)",
55
"mimeType": "text/markdown"
66
},
77
"userLabels": {

‎alerts/google-gke/grpc-server-errors-entire-workload.v1.json‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"displayName": "Server Error is high. (GKE Workload: ${WORKLOAD_NAME})",
33
"documentation": {
4-
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${CLUSTER_LOCATION}/${CLUSTER_NAME}/${NAMESPACE}/${WORKLOAD_NAME}/observability?project=${PROJECT_ID}).\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [MQL documentation](https://cloud.google.com/monitoring/mql/alerts)",
4+
"content": "[View Workload Details](https://console.cloud.google.com/kubernetes/deployment/${metric_or_resource.label.location}/${metric_or_resource.label.cluster_name}/${metric_or_resource.label.namespace_name}/${metric.label.metadata_system_top_level_controller_name}/observability?project=${metric_or_resource.label.project_id}).\n- If alerts tend to be false positive or noisy, consider visiting the alert policy page and changing the threshold, the rolling (alignment) window, and the retest (duration) window. [Alerting policies documentation](https://cloud.google.com/monitoring/alerts/concepts-indepth), [PromQL documentation](https://cloud.google.com/monitoring/promql/create-promql-alerts-console)",
55
"mimeType": "text/markdown"
66
},
77
"userLabels": {
@@ -15,19 +15,22 @@
1515
"conditions": [
1616
{
1717
"displayName": "Server Error is high for a method",
18-
"conditionMonitoringQueryLanguage": {
18+
"conditionPrometheusQueryLanguage": {
1919
"duration": "0s",
20-
"trigger": {
21-
"count": 1
22-
},
23-
"query": "fetch prometheus_target\n| metric 'prometheus.googleapis.com/grpc_server_handled_total/counter'\n| filter\n resource.project_id == '${PROJECT_ID}' &&\n resource.cluster == '${CLUSTER_NAME}' &&\n resource.location == '${CLUSTER_LOCATION}' &&\n resource.namespace == '${NAMESPACE}' &&\n metric.pod =~ '${WORKLOAD_NAME}-[a-f0-9]{7,10}-[a-z0-9]{5}'\n|\n{\n filter metric.grpc_code =~ '(Internal|Unknown|Unavailable|Unimplemented)'\n | align rate(5m)\n | every 1m\n | group_by [metric.grpc_method], [value_grpc_server_handled_total_aggregate: aggregate(value.grpc_server_handled_total)]\n ;\n align rate(5m)\n | every 1m\n | group_by [metric.grpc_method], [value_grpc_server_handled_total_aggregate: aggregate(value.grpc_server_handled_total)]\n}\n| join\n| div\n| cast_units('1')\n| scale('%')\n| condition val() > 5'%'"
20+
"evaluationInterval": "30s",
21+
"query": "(\n sum by (grpc_method) (\n rate({\n \"grpc_server_handled_total\",\n project_id=~\"${PROJECT_ID}\",\n cluster=~\"${CLUSTER_NAME}\",\n location=~\"${CLUSTER_LOCATION}\",\n namespace=~\"${NAMESPACE}\",\n pod=~\"${WORKLOAD_NAME}-[a-f0-9]{7,10}-[a-z0-9]{5}\",\n grpc_code=~\"(Internal|Unknown|Unavailable|Unimplemented)\"\n }[5m])\n )\n /\n sum by (grpc_method) (\n rate({\n \"grpc_server_handled_total\",\n project_id=~\"${PROJECT_ID}\",\n cluster=~\"${CLUSTER_NAME}\",\n location=~\"${CLUSTER_LOCATION}\",\n namespace=~\"${NAMESPACE}\",\n pod=~\"${WORKLOAD_NAME}-[a-f0-9]{7,10}-[a-z0-9]{5}\"\n }[5m])\n )\n) > 0.05"
2422
}
2523
}
2624
],
2725
"alertStrategy": {
28-
"autoClose": "604800s"
26+
"autoClose": "604800s",
27+
"notificationPrompts": [
28+
"OPENED",
29+
"CLOSED"
30+
]
2931
},
3032
"combiner": "OR",
3133
"enabled": true,
32-
"notificationChannels": []
34+
"notificationChannels": [],
35+
"severity": "SEVERITY_UNSPECIFIED"
3336
}

0 commit comments

Comments
 (0)