Skip to content

Commit bb29ed3

Browse files
OscarLlamas6OscarLlamas6
authored
Merge pull request #40 from datum-cloud/feature/nso-slo
Defining Gateways SLOs w/ Alerting
2 parents 7512ff9 + 28d022d commit bb29ed3

8 files changed

Lines changed: 282 additions & 3 deletions

File tree

‎.github/workflows/validate-kustomize.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
name: Validate Kustomize Configurations
2+
3+
on:
4+
push:
5+
6+
jobs:
7+
validate-kustomize:
8+
uses: datum-cloud/actions/.github/workflows/validate-kustomize.yaml@v1.6.1

‎Taskfile.yaml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
version: '3'
2+
3+
tasks:
4+
validate-kustomizations:
5+
desc: Validate all kustomization.yaml files using kustomize build
6+
cmds:
7+
- echo "# Kustomize Validation Results"
8+
- echo ""
9+
- |
10+
HAS_ERRORS=0
11+
KUSTOMIZATION_DIRS=$(find . -name "kustomization.yaml" -exec dirname {} \;)
12+
for dir in $KUSTOMIZATION_DIRS; do
13+
echo "🔍 Validating: $dir"
14+
if ! OUTPUT=$(kustomize build "$dir" --enable-helm 2>&1); then
15+
echo ""
16+
echo "❌ Error in '$dir':"
17+
echo "----------------------------------------"
18+
echo "$OUTPUT"
19+
echo "----------------------------------------"
20+
echo ""
21+
HAS_ERRORS=1
22+
else
23+
echo "✅ $dir is valid"
24+
fi
25+
done
26+
if [ "$HAS_ERRORS" -eq 1 ]; then
27+
echo ""
28+
echo "🚨 One or more kustomizations failed validation."
29+
exit 1
30+
else
31+
echo ""
32+
echo "🎉 All kustomizations are valid."
33+
fi
34+
silent: false
35+
36+
test-prometheus-rules:
37+
desc: Run unit tests for Prometheus alerting rules
38+
cmds:
39+
- echo "# Prometheus Rules Test Results"
40+
- echo ""
41+
- |
42+
HAS_ERRORS=0
43+
TEST_FILES=$(find test/prometheus-rules -name "*-tests.yaml" 2>/dev/null)
44+
45+
if [ -z "$TEST_FILES" ]; then
46+
echo "⚠️ No Prometheus test files found in test/prometheus-rules/"
47+
exit 0
48+
fi
49+
50+
for test_file in $TEST_FILES; do
51+
test_dir=$(dirname "$test_file")
52+
test_name=$(basename "$test_file")
53+
echo "🔍 Testing: $test_file"
54+
55+
if ! OUTPUT=$(cd "$test_dir" && promtool test rules "$test_name" 2>&1); then
56+
echo ""
57+
echo "❌ Test failed for '$test_file':"
58+
echo "----------------------------------------"
59+
echo "$OUTPUT"
60+
echo "----------------------------------------"
61+
echo ""
62+
HAS_ERRORS=1
63+
else
64+
echo "✅ $test_file passed"
65+
fi
66+
done
67+
68+
if [ "$HAS_ERRORS" -eq 1 ]; then
69+
echo ""
70+
echo "🚨 One or more Prometheus rule tests failed."
71+
exit 1
72+
else
73+
echo ""
74+
echo "🎉 All Prometheus rule tests passed."
75+
fi
76+
silent: false

‎config/default/kustomization.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ namePrefix: network-services-operator-
1414
# pairs:
1515
# someName: someValue
1616

17+
components:
18+
- ../resource-metrics
19+
1720
resources:
1821
- ../crd
1922
- ../rbac
2023
- ../manager
21-
- ../resource-metrics
2224
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
2325
# crd/kustomization.yaml
2426
- ../webhook
@@ -35,11 +37,11 @@ resources:
3537
#- ../network-policy
3638

3739
# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager
38-
patches:
40+
#patches:
3941

4042
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
4143
# crd/kustomization.yaml
42-
- path: manager_webhook_patch.yaml
44+
#- path: manager_webhook_patch.yaml
4345

4446
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
4547
# Uncomment the following replacements to add the cert-manager CA injection annotations

‎config/resource-metrics/gateways.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,17 @@ spec:
5151
labelsFromPath:
5252
type: ["type"]
5353
valueFrom: ["status"]
54+
- name: "status_condition_last_transition_time"
55+
help: "last transition time for status conditions"
56+
each:
57+
type: Gauge
58+
gauge:
59+
path: [status, conditions]
60+
labelsFromPath:
61+
type: ["type"]
62+
reason: ["reason"]
63+
status: ["status"]
64+
valueFrom: ["lastTransitionTime"]
5465
- name: "custom_hostname"
5566
help: "Custom hostname defined on the gateway"
5667
errorLogV: 10

‎config/telemetry/alerts/gateways.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: PrometheusRule
3+
metadata:
4+
name: nso-slo
5+
namespace: nso-slo
6+
spec:
7+
groups:
8+
- name: nso-slo
9+
interval: 30s
10+
rules:
11+
- alert: GatewayNotReadySLOViolation
12+
expr: |
13+
sum(time() - datum_cloud_networking_gateway_created) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) > 60
14+
unless (
15+
sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) == 2
16+
)
17+
for: 0s
18+
labels:
19+
severity: critical
20+
slo_violation: "true"
21+
annotations:
22+
summary: "Gateway {{ $labels.resource_name }} is taking longer than 60 seconds to reach Ready status"
23+
description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in creation state for {{ $value }} seconds without reaching Ready status (Accepted=True AND Programmed=True), which exceeds the 60-second SLO threshold."
24+
25+
- alert: GatewayDegradedSLOViolation
26+
expr: |
27+
(
28+
sum(time() - datum_cloud_networking_gateway_status_condition_last_transition_time{type=~"Accepted|Programmed", status="False"})
29+
by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name)
30+
) > 60
31+
and (
32+
sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1)
33+
by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) < 2
34+
)
35+
for: 0s
36+
labels:
37+
severity: critical
38+
slo_violation: "true"
39+
annotations:
40+
summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
41+
description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."

‎config/telemetry/alerts/kustomization.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: kustomize.config.k8s.io/v1alpha1
2+
kind: Component
3+
4+
resources:
5+
- gateways.yaml

‎test/prometheus-rules/gateways/nso-slo-rules.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
groups:
2+
- name: nso-slo
3+
interval: 30s
4+
rules:
5+
- alert: GatewayNotReadySLOViolation
6+
expr: |
7+
sum(time() - datum_cloud_networking_gateway_created) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) > 60
8+
unless (
9+
sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) == 2
10+
)
11+
for: 0s
12+
labels:
13+
severity: critical
14+
slo_violation: "true"
15+
annotations:
16+
summary: "Gateway {{ $labels.resource_name }} is taking longer than 60 seconds to reach Ready status"
17+
description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in creation state for {{ $value }} seconds without reaching Ready status (Accepted=True AND Programmed=True), which exceeds the 60-second SLO threshold."
18+
19+
- alert: GatewayDegradedSLOViolation
20+
expr: |
21+
(
22+
sum(time() - datum_cloud_networking_gateway_status_condition_last_transition_time{type=~"Accepted|Programmed", status="False"})
23+
by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name)
24+
) > 60
25+
and (
26+
sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1)
27+
by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) < 2
28+
)
29+
for: 0s
30+
labels:
31+
severity: critical
32+
slo_violation: "true"
33+
annotations:
34+
summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
35+
description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."

‎test/prometheus-rules/gateways/nso-slo-tests.yaml‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
rule_files:
2+
- nso-slo-rules.yaml
3+
4+
evaluation_interval: 30s
5+
6+
tests:
7+
# Test for GatewayNotReadySLOViolation - Gateway created more than 60s ago but not ready
8+
- interval: 1m
9+
input_series:
10+
# Gateway created 120 seconds ago (eval_time 60s - 120s = timestamp -60)
11+
- series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway"}'
12+
values: '-60+0x3'
13+
# Only has Accepted=1 status, missing Programmed
14+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway", type="Accepted"}'
15+
values: '1+0x3'
16+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway", type="Programmed"}'
17+
values: '0+0x3'
18+
alert_rule_test:
19+
- eval_time: 1m
20+
alertname: GatewayNotReadySLOViolation
21+
exp_alerts:
22+
- exp_labels:
23+
severity: critical
24+
slo_violation: "true"
25+
resourcemanager_datumapis_com_project_name: test-project
26+
resource_namespace: test-ns
27+
resource_name: test-gateway
28+
exp_annotations:
29+
summary: "Gateway test-gateway is taking longer than 60 seconds to reach Ready status"
30+
description: "Gateway test-gateway in namespace test-ns has been in creation state for 120 seconds without reaching Ready status (Accepted=True AND Programmed=True), which exceeds the 60-second SLO threshold."
31+
32+
# Test for GatewayNotReadySLOViolation - Gateway ready (should NOT alert)
33+
- interval: 1m
34+
input_series:
35+
# Gateway created 120 seconds ago
36+
- series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway"}'
37+
values: '-60+0x3'
38+
# Both statuses are 1 (Ready)
39+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway", type="Accepted"}'
40+
values: '1+0x3'
41+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway", type="Programmed"}'
42+
values: '1+0x3'
43+
alert_rule_test:
44+
- eval_time: 1m
45+
alertname: GatewayNotReadySLOViolation
46+
exp_alerts: []
47+
48+
# Test for GatewayDegradedSLOViolation - Gateway degraded for more than 60s
49+
- interval: 1m
50+
input_series:
51+
# Transition to False 120 seconds ago (eval_time 60s - 120s = timestamp -60)
52+
- series: 'datum_cloud_networking_gateway_status_condition_last_transition_time{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Accepted", status="False"}'
53+
values: '-60+0x3'
54+
# Current status is False (0)
55+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Accepted"}'
56+
values: '0+0x3'
57+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Programmed"}'
58+
values: '1+0x3'
59+
alert_rule_test:
60+
- eval_time: 1m
61+
alertname: GatewayDegradedSLOViolation
62+
exp_alerts:
63+
- exp_labels:
64+
severity: critical
65+
slo_violation: "true"
66+
resourcemanager_datumapis_com_project_name: test-project
67+
resource_namespace: test-ns
68+
resource_name: degraded-gateway
69+
exp_annotations:
70+
summary: "Gateway degraded-gateway has been degraded for over 60 seconds"
71+
description: "Gateway degraded-gateway in namespace test-ns has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
72+
73+
# Test for GatewayDegradedSLOViolation - Gateway functional (should NOT alert)
74+
- interval: 1m
75+
input_series:
76+
# Both statuses are 1 (functional)
77+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="healthy-gateway", type="Accepted"}'
78+
values: '1+0x3'
79+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="healthy-gateway", type="Programmed"}'
80+
values: '1+0x3'
81+
alert_rule_test:
82+
- eval_time: 1m
83+
alertname: GatewayDegradedSLOViolation
84+
exp_alerts: []
85+
86+
# Edge case test - Gateway created exactly 60s ago (should NOT alert)
87+
- interval: 1m
88+
input_series:
89+
# Gateway created exactly 60 seconds ago (time() - created = 60)
90+
# If eval_time is 1m (60s), then created should be 0
91+
- series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway"}'
92+
values: '0+0x3'
93+
# Only has Accepted
94+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway", type="Accepted"}'
95+
values: '1+0x3'
96+
- series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway", type="Programmed"}'
97+
values: '0+0x3'
98+
alert_rule_test:
99+
- eval_time: 1m
100+
alertname: GatewayNotReadySLOViolation
101+
exp_alerts: []

0 commit comments

Comments
 (0)