Skip to content

Commit def2026

Browse files
sshcherbakovsshcherbakov
authored
Add alert policy example for monitoring Google Cloud KMS service quotas consumption (#1232)
1 parent 0ad8dc7 commit def2026

4 files changed

Lines changed: 172 additions & 0 deletions

File tree

‎alerts/google-cloud-kms/README.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Alert policies for Cloud KMS service monitoring
2+
3+
This contains alert policies and Terraform script to deploy them for [new generation](https://docs.cloud.google.com/kms/quotas#new-quotas) Cloud KMS quotas alert polciies.
4+
5+
The following alert policies are included:
6+
- Key Read Usage quota metric `cloudkms.googleapis.com​/read_usage`
7+
- Key Write Usage quota metric `cloudkms.googleapis.com​/write_usage`
8+
- Software Key Crypto quota metric `cloudkms.googleapis.com​/software_usage`
9+
- Hardware Key Crypto quota metric `cloudkms.googleapis.com​/hsm`
10+
- External Key Crypto quota metric `cloudkms.googleapis.com​/external_kms_usage`
11+
12+
The alert policies are created for each location specified in the `location` variable. A separate alert policy is created for each quota metric.
13+
14+
Example of alert policy for "Write Usage" quota metric in "europe-west3" location:
15+
```
16+
{
17+
"displayName": "Quota usage - Cloud KMS API - Token usage per minute per region - cloudkms.googleapis.com/write_usage (europe-west3)",
18+
"documentation": {
19+
"content": "To resolve this issue, click [here](https://console.cloud.google.com/iam-admin/quotas?service=${resource.label.service}&metric=${metric.label.quota_metric}&limit=${metric.label.limit_name}&dimensions=%7B%22region%22%3A%22${resource.label.location}%22%7D&project=PROJET_ID&fromNotifications=1) to review quota usage and optionally request a quota increase.",
20+
"mimeType": "text/markdown"
21+
},
22+
"conditions": [
23+
{
24+
"displayName": "Quota usage reached defined threshold",
25+
"conditionThreshold": {
26+
"aggregations": [
27+
{
28+
"alignmentPeriod": "60s",
29+
"perSeriesAligner": "ALIGN_RATE"
30+
}
31+
],
32+
"comparison": "COMPARISON_GT",
33+
"duration": "60s",
34+
"filter": "metric.type = \"serviceruntime.googleapis.com/quota_rate_net_usage\" AND metric.label.quota_metric = \"cloudkms.googleapis.com/write_usage\" AND resource.type = \"consumer_quota\" AND resource.label.location = \"europe-west3\" AND resource.label.quota_metric = \"cloudkms.googleapis.com/write_usage\" AND resource.label.limit_name = \"WriteTokensUsagePerMinutePerProjectPerRegion\" AND metric.label.limit_name = \"WriteTokensUsagePerMinutePerProjectPerRegion\" AND metric.label.project_id = \"PROJET_ID\" AND resource.label.project_id = \"PROJET_ID\"",
35+
"thresholdValue": 544,
36+
"trigger": {
37+
"count": 1
38+
}
39+
}
40+
}
41+
],
42+
"alertStrategy": {
43+
"autoClose": "604800s"
44+
},
45+
"combiner": "OR",
46+
"enabled": true
47+
}
48+
```

‎alerts/google-cloud-kms/cloud-kms-alert-policies.tf‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
locals {
2+
alert_policies = {
3+
for pair in setproduct(var.location, keys(var.quota_metrics)) :
4+
"${pair[0]}-${pair[1]}" => {
5+
location = pair[0]
6+
quota_metric = pair[1]
7+
limit_name = var.quota_metrics[pair[1]]
8+
}
9+
}
10+
}
11+
12+
resource "google_monitoring_alert_policy" "cloud_kms_quota_usage" {
13+
for_each = local.alert_policies
14+
project = var.project_id
15+
display_name = "Quota usage - Cloud KMS API - Token usage per minute per region - ${each.value.quota_metric} (${each.value.location})"
16+
combiner = "OR"
17+
enabled = true
18+
19+
conditions {
20+
display_name = "Quota usage reached defined threshold"
21+
condition_prometheus_query_language {
22+
duration = var.alert_duration
23+
evaluation_interval = var.alert_evaluation_interval
24+
query = <<-EOT
25+
max_over_time(
26+
sum(
27+
label_replace(
28+
increase(
29+
serviceruntime_googleapis_com:quota_rate_net_usage{
30+
project_id='${var.project_id}',
31+
monitored_resource='consumer_quota',
32+
quota_metric='${each.value.quota_metric}',
33+
location='${each.value.location}'
34+
}[1m]
35+
),
36+
'limit_name', '${each.value.limit_name}', '', ''
37+
)
38+
) by (service, limit_name, project_id, quota_metric, location)[4m:1m]
39+
)
40+
/
41+
min(
42+
${each.value.quota_metric == "cloudkms.googleapis.com/external_kms_usage" ? 60 : 1} * last_over_time(
43+
serviceruntime_googleapis_com:quota_limit{
44+
project_id='${var.project_id}',
45+
monitored_resource='consumer_quota',
46+
quota_metric='${each.value.quota_metric}',
47+
location='${each.value.location}',
48+
limit_name='${each.value.limit_name}'
49+
}[1d]
50+
)
51+
) by (service, limit_name, project_id, quota_metric, location)
52+
> ${var.alert_threshold}
53+
EOT
54+
}
55+
}
56+
57+
documentation {
58+
content = "To resolve this issue, click [here](https://console.cloud.google.com/iam-admin/quotas?service=$${resource.label.service}&metric=$${metric.label.quota_metric}&limit=$${metric.label.limit_name}&dimensions=%7B%22region%22%3A%22$${resource.label.location}%22%7D&project=${var.project_id}&fromNotifications=1) to review quota usage and optionally request a quota increase."
59+
mime_type = "text/markdown"
60+
}
61+
62+
alert_strategy {
63+
auto_close = var.alert_auto_close
64+
notification_prompts = ["OPENED"]
65+
}
66+
67+
notification_channels = var.notification_channels
68+
}

‎alerts/google-cloud-kms/terraform.tfvars.sample‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
project_id = "my-project-id"
2+
location = ["europe-west3"]
3+
notification_channels = [
4+
"projects/my-project-id/notificationChannels/1234567890"
5+
]

‎alerts/google-cloud-kms/variables.tf‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
variable "project_id" {
2+
description = "Target Project ID"
3+
type = string
4+
}
5+
6+
variable "location" {
7+
description = "Cloud KMS resources location"
8+
type = list(string)
9+
default = ["europe-west3"]
10+
}
11+
12+
variable "notification_channels" {
13+
description = "List of notification channels to attach to alert policies"
14+
type = list(string)
15+
}
16+
17+
variable "quota_metrics" {
18+
description = "Map of Cloud KMS quota metrics to their corresponding limit names"
19+
type = map(string)
20+
default = {
21+
"cloudkms.googleapis.com/write_usage" = "WriteTokensUsagePerMinutePerProjectPerRegion"
22+
"cloudkms.googleapis.com/read_usage" = "ReadTokensUsagePerMinutePerProjectPerRegion"
23+
"cloudkms.googleapis.com/software_usage" = "SoftwareTokensUsagePerMinutePerProjectPerRegion"
24+
"cloudkms.googleapis.com/hsm_usage" = "HsmTokensUsagePerMinutePerProjectPerRegion"
25+
"cloudkms.googleapis.com/external_kms_usage" = "ExternalTokensUsagePerMinutePerProjectPerRegion"
26+
}
27+
}
28+
29+
variable "alert_duration" {
30+
description = "The duration for which the condition must be met before an alert is triggered"
31+
type = string
32+
default = "60s"
33+
}
34+
35+
variable "alert_evaluation_interval" {
36+
description = "The interval at which the condition is evaluated"
37+
type = string
38+
default = "60s"
39+
}
40+
41+
variable "alert_auto_close" {
42+
description = "The duration after which an open incident is automatically closed"
43+
type = string
44+
default = "604800s"
45+
}
46+
47+
variable "alert_threshold" {
48+
description = "The threshold for the quota usage alert (fraction of limit)"
49+
type = number
50+
default = 0.8
51+
}

0 commit comments

Comments
 (0)