Skip to content

Commit ed074d9

Browse files
yqluyqlu
authored
Add OTel GenAI agent alerts (#1235)
1 parent 9e45fbb commit ed074d9

7 files changed

Lines changed: 98 additions & 4 deletions

‎alerts/google-vertex-ai/agent-runtime-high-error-rate.v1.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"displayName": "Agent Runtime - High Error Rate (${REASONING_ENGINE_ID})",
33
"documentation": {
4-
"content": "This alert fires when the error rate of the agent runtime instance (${REASONING_ENGINE_ID}) rises above 5% for 5 minutes or more.",
4+
"content": "This alert fires when the error rate (calculated as the percentage of requests with a non-2xx status code) of the Agent Runtime instance (${REASONING_ENGINE_ID}) rises above 5% for 5 minutes or more.",
55
"mimeType": "text/markdown"
66
},
77
"userLabels": {

‎alerts/google-vertex-ai/agent-runtime-high-latency.v1.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"displayName": "Agent Runtime - High P95 Request Latency (${REASONING_ENGINE_ID})",
33
"documentation": {
4-
"content": "This alert fires when the P95 request latency of the agent runtime instance (${REASONING_ENGINE_ID}) rises above 120s for 5 minutes or more.",
4+
"content": "This alert fires when the P95 request latency of the Agent Runtime instance (${REASONING_ENGINE_ID}) rises above 120s for 5 minutes or more.",
55
"mimeType": "text/markdown"
66
},
77
"userLabels": {

‎alerts/google-vertex-ai/metadata.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@ alert_policy_templates:
2929
platform: GCP
3030
- id: "agent-runtime-high-latency"
3131
display_name: "Agent Runtime High P95 Request Latency"
32-
description: "This alert fires when the P95 request latency of the agent runtime instance rises above 120s for 5 minutes or more."
32+
description: "This alert fires when the P95 request latency of the Agent Runtime instance rises above 120s for 5 minutes or more."
3333
version: 1
3434
related_integrations:
3535
- id: vertex_ai
3636
platform: GCP
3737
- id: "agent-runtime-high-error-rate"
3838
display_name: "Agent Runtime High Error Rate"
39-
description: "This alert fires when the error rate of the agent runtime instance rises above 5% for 5 minutes or more."
39+
description: "This alert fires when the error rate (calculated as the percentage of requests with a non-2xx status code) of the Agent Runtime instance rises above 5% for 5 minutes or more."
4040
version: 1
4141
related_integrations:
4242
- id: vertex_ai

‎alerts/otel-genai/metadata.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
alert_policy_templates:
2+
- id: "otel-genai-agent-token-usage-high"
3+
display_name: "Agent Token Usage High"
4+
description: "This alert fires when token usage (over both input / output) exceeds 1 million over the past 60 mins. Edit the thresholds to meet your needs."
5+
version: 1
6+
- id: "otel-genai-agent-tool-error-rate"
7+
display_name: "Agent to Tool Error Rate High"
8+
description: "This alert will be triggered if the agent has a > 5% error rate in invoking a given tool for 5 mins or more."
9+
version: 1
10+
- id: "otel-genai-agent-model-error-rate"
11+
display_name: "Agent to Model Error Rate High"
12+
description: "This alert will be triggered if the agent has a > 5% error rate in invoking a given model for 5 mins or more."
13+
version: 1

‎alerts/otel-genai/otel-genai-agent-model-error-rate.v1.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"displayName": "Agent - High Tool Call Error Rate (${AGENT_NAME})",
3+
"documentation": {
4+
"content": "This alert will be triggered if the agent has a > 5% error rate in invoking a given tool for 5 mins or more.",
5+
"mimeType": "text/markdown"
6+
},
7+
"userLabels": {
8+
"agent_display_name": "${AGENT_NAME}",
9+
"agent_id": "${AGENT_ID}",
10+
"location": "${LOCATION}"
11+
},
12+
"conditions": [
13+
{
14+
"displayName": "Agent - High Model Call Error Rate (${AGENT_NAME})",
15+
"conditionPrometheusQueryLanguage": {
16+
"duration": "0s",
17+
"evaluationInterval": "30s",
18+
"query": "sum by (\"gen_ai.response.model\")(\n increase({\n \"__name__\"=\"gen_ai.client.operation.duration_count\",\n \"location\"=\"${LOCATION}\",\n \"cluster\"=\"${AGENT_ID}\",\n \"error.type\"!= \"\"\n }[5m])\n)\n/\nsum by (\"gen_ai.response.model\")(\n increase({\n \"__name__\"=\"gen_ai.client.operation.duration_count\",\n \"location\"=\"${LOCATION}\",\n \"cluster\"=\"${AGENT_ID}\"\n }[5m])\n) > 0.05\n"
19+
}
20+
}
21+
],
22+
"alertStrategy": {
23+
"autoClose": "604800s"
24+
},
25+
"combiner": "OR",
26+
"enabled": true
27+
}

‎alerts/otel-genai/otel-genai-agent-token-usage-high.v1.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"displayName": "Agent - High Token Usage (${AGENT_NAME})",
3+
"documentation": {
4+
"content": "This alert fires when token usage (over both input and output) exceeds 1 million tokens over the past hour.",
5+
"mimeType": "text/markdown"
6+
},
7+
"userLabels": {
8+
"agent_display_name": "${AGENT_NAME}",
9+
"agent_id": "${AGENT_ID}",
10+
"location": "${LOCATION}"
11+
},
12+
"conditions": [
13+
{
14+
"displayName": "Agent - High Token Usage (${AGENT_NAME})",
15+
"conditionPrometheusQueryLanguage": {
16+
"duration": "300s",
17+
"evaluationInterval": "60s",
18+
"query": "sum (increase({\n \"__name__\"=\"gen_ai.client.token.usage_sum\",\n \"location\"=\"${LOCATION}\",\n \"cluster\"=\"${AGENT_ID}\", \n \"gen_ai.token.type\"=\"input\"\n}[1m])) + \nsum(increase({\n \"__name__\"=\"gen_ai.client.token.usage_sum\", \n \"location\"=\"${LOCATION}\",\n \"cluster\"=\"${AGENT_ID}\", \n \"gen_ai.token.type\"=\"output\"\n}[1m])) > 1000000"
19+
}
20+
}
21+
],
22+
"alertStrategy": {
23+
"autoClose": "604800s"
24+
},
25+
"combiner": "OR",
26+
"enabled": true
27+
}

‎alerts/otel-genai/otel-genai-agent-tool-error-rate.v1.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"displayName": "Agent - High Tool Call Error Rate (${AGENT_NAME})",
3+
"documentation": {
4+
"content": "This alert will be triggered if the agent has a > 5% error rate in invoking a given tool for 5 mins or more.",
5+
"mimeType": "text/markdown"
6+
},
7+
"userLabels": {
8+
"agent_display_name": "${AGENT_NAME}",
9+
"agent_id": "${AGENT_ID}",
10+
"location": "${LOCATION}"
11+
},
12+
"conditions": [
13+
{
14+
"displayName": "Agent - High Tool Call Error Rate (${AGENT_NAME})",
15+
"conditionPrometheusQueryLanguage": {
16+
"duration": "0s",
17+
"evaluationInterval": "30s",
18+
"query": "sum by (\"gen_ai.tool.name\")(\n increase({\n \"__name__\"=\"gen_ai.tool.execution.duration_count\", \n \"location\"=\"${LOCATION}\",\n \"cluster\"=\"${AGENT_ID}\",\n \"error.type\"!= \"\"}[5m]\n )\n)\n/\nsum by (\"gen_ai.tool.name\")(\n increase({\n \"__name__\"=\"gen_ai.tool.execution.duration_count\", \n \"location\"=\"${LOCATION}\",\n \"cluster\"=\"${AGENT_ID}\",\n \"error.type\"!= \"\"}[5m]\n )\n) > 0.05\n"
19+
}
20+
}
21+
],
22+
"alertStrategy": {
23+
"autoClose": "604800s"
24+
},
25+
"combiner": "OR",
26+
"enabled": true
27+
}

0 commit comments

Comments
 (0)