Skip to content

Commit c9a3564

Browse files
OscarLlamas6OscarLlamas6
committed
chore: add unit testing for Gateway PrometheusRules
1 parent c3f7684 commit c9a3564

2 files changed

Lines changed: 136 additions & 0 deletions

File tree

‎test/prometheus-rules/gateways/nso-slo-rules.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
groups:
2+
- name: nso-slo
3+
interval: 30s
4+
rules:
5+
- alert: GatewayNotReadySLOViolation
6+
expr: |
7+
sum(time() - datum_cloud_networking_gateway_created) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) > 60
8+
unless (
9+
sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) == 2
10+
)
11+
for: 0s
12+
labels:
13+
severity: critical
14+
slo_violation: "true"
15+
annotations:
16+
summary: "Gateway {{ $labels.resource_name }} is taking longer than 60 seconds to reach Ready status"
17+
description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in creation state for {{ $value }} seconds without reaching Ready status (Accepted=True AND Programmed=True), which exceeds the 60-second SLO threshold."
18+
19+
- alert: GatewayDegradedSLOViolation
20+
expr: |
21+
(
22+
sum(time() - datum_cloud_networking_gateway_status_condition_last_transition_time{type=~"Accepted|Programmed", status="False"})
23+
by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name)
24+
) > 60
25+
and (
26+
sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1)
27+
by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) < 2
28+
)
29+
for: 0s
30+
labels:
31+
severity: critical
32+
slo_violation: "true"
33+
annotations:
34+
summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
35+
description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."

‎test/prometheus-rules/gateways/nso-slo-tests.yaml‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
rule_files:
2+
- nso-slo-rules.yaml
3+
4+
evaluation_interval: 30s
5+
6+
tests:
7+
# Test for GatewayNotReadySLOViolation - Gateway created more than 60s ago but not ready
8+
- interval: 1m
9+
input_series:
10+
# Gateway created 120 seconds ago (eval_time 60s - 120s = timestamp -60)
11+
- series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway"}'
12+
values: '-60+0x3'
13+
# Only has Accepted=1 status, missing Programmed
14+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway", type="Accepted"}'
15+
values: '1+0x3'
16+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway", type="Programmed"}'
17+
values: '0+0x3'
18+
alert_rule_test:
19+
- eval_time: 1m
20+
alertname: GatewayNotReadySLOViolation
21+
exp_alerts:
22+
- exp_labels:
23+
severity: critical
24+
slo_violation: "true"
25+
resourcemanager_datumapis_com_project_name: test-project
26+
resource_namespace: test-ns
27+
resource_name: test-gateway
28+
exp_annotations:
29+
summary: "Gateway test-gateway is taking longer than 60 seconds to reach Ready status"
30+
description: "Gateway test-gateway in namespace test-ns has been in creation state for 120 seconds without reaching Ready status (Accepted=True AND Programmed=True), which exceeds the 60-second SLO threshold."
31+
32+
# Test for GatewayNotReadySLOViolation - Gateway ready (should NOT alert)
33+
- interval: 1m
34+
input_series:
35+
# Gateway created 120 seconds ago
36+
- series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway"}'
37+
values: '-60+0x3'
38+
# Both statuses are 1 (Ready)
39+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway", type="Accepted"}'
40+
values: '1+0x3'
41+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway", type="Programmed"}'
42+
values: '1+0x3'
43+
alert_rule_test:
44+
- eval_time: 1m
45+
alertname: GatewayNotReadySLOViolation
46+
exp_alerts: []
47+
48+
# Test for GatewayDegradedSLOViolation - Gateway degraded for more than 60s
49+
- interval: 1m
50+
input_series:
51+
# Transition to False 120 seconds ago (eval_time 60s - 120s = timestamp -60)
52+
- series: 'datum_cloud_networking_gateway_status_condition_last_transition_time{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Accepted", status="False"}'
53+
values: '-60+0x3'
54+
# Current status is False (0)
55+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Accepted"}'
56+
values: '0+0x3'
57+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Programmed"}'
58+
values: '1+0x3'
59+
alert_rule_test:
60+
- eval_time: 1m
61+
alertname: GatewayDegradedSLOViolation
62+
exp_alerts:
63+
- exp_labels:
64+
severity: critical
65+
slo_violation: "true"
66+
resourcemanager_datumapis_com_project_name: test-project
67+
resource_namespace: test-ns
68+
resource_name: degraded-gateway
69+
exp_annotations:
70+
summary: "Gateway degraded-gateway has been degraded for over 60 seconds"
71+
description: "Gateway degraded-gateway in namespace test-ns has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
72+
73+
# Test for GatewayDegradedSLOViolation - Gateway functional (should NOT alert)
74+
- interval: 1m
75+
input_series:
76+
# Both statuses are 1 (functional)
77+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="healthy-gateway", type="Accepted"}'
78+
values: '1+0x3'
79+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="healthy-gateway", type="Programmed"}'
80+
values: '1+0x3'
81+
alert_rule_test:
82+
- eval_time: 1m
83+
alertname: GatewayDegradedSLOViolation
84+
exp_alerts: []
85+
86+
# Edge case test - Gateway created exactly 60s ago (should NOT alert)
87+
- interval: 1m
88+
input_series:
89+
# Gateway created exactly 60 seconds ago (time() - created = 60)
90+
# If eval_time is 1m (60s), then created should be 0
91+
- series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway"}'
92+
values: '0+0x3'
93+
# Only has Accepted
94+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway", type="Accepted"}'
95+
values: '1+0x3'
96+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway", type="Programmed"}'
97+
values: '0+0x3'
98+
alert_rule_test:
99+
- eval_time: 1m
100+
alertname: GatewayNotReadySLOViolation
101+
exp_alerts: []

0 commit comments

Comments
 (0)