Defining Gateways SLOs w/ Alerting

.github/workflows/validate-kustomize.yaml

@@ -0,0 +1,8 @@
+name: Validate Kustomize Configurations
+
+on:
+  push:
+
+jobs:
+  validate-kustomize:
+    uses: datum-cloud/actions/.github/workflows/validate-kustomize.yaml@v1.6.1

Taskfile.yaml

@@ -0,0 +1,76 @@
+version: '3'
+
+tasks:
+  validate-kustomizations:
+      desc: Validate all kustomization.yaml files using kustomize build
+      cmds:
+        - echo "# Kustomize Validation Results"
+        - echo ""
+        - |
+          HAS_ERRORS=0
+          KUSTOMIZATION_DIRS=$(find . -name "kustomization.yaml" -exec dirname {} \;)
+          for dir in $KUSTOMIZATION_DIRS; do
+            echo "🔍 Validating: $dir"
+            if ! OUTPUT=$(kustomize build "$dir" --enable-helm 2>&1); then
+              echo ""
+              echo "❌ Error in '$dir':"
+              echo "----------------------------------------"
+              echo "$OUTPUT"
+              echo "----------------------------------------"
+              echo ""
+              HAS_ERRORS=1
+            else
+              echo "✅ $dir is valid"
+            fi
+          done
+          if [ "$HAS_ERRORS" -eq 1 ]; then
+            echo ""
+            echo "🚨 One or more kustomizations failed validation."
+            exit 1
+          else
+            echo ""
+            echo "🎉 All kustomizations are valid."
+          fi
+      silent: false
+
+  test-prometheus-rules:
+      desc: Run unit tests for Prometheus alerting rules
+      cmds:
+        - echo "# Prometheus Rules Test Results"
+        - echo ""
+        - |
+          HAS_ERRORS=0
+          TEST_FILES=$(find test/prometheus-rules -name "*-tests.yaml" 2>/dev/null)
+
+          if [ -z "$TEST_FILES" ]; then
+            echo "⚠️  No Prometheus test files found in test/prometheus-rules/"
+            exit 0
+          fi
+
+          for test_file in $TEST_FILES; do
+            test_dir=$(dirname "$test_file")
+            test_name=$(basename "$test_file")
+            echo "🔍 Testing: $test_file"
+
+            if ! OUTPUT=$(cd "$test_dir" && promtool test rules "$test_name" 2>&1); then
+              echo ""
+              echo "❌ Test failed for '$test_file':"
+              echo "----------------------------------------"
+              echo "$OUTPUT"
+              echo "----------------------------------------"
+              echo ""
+              HAS_ERRORS=1
+            else
+              echo "✅ $test_file passed"
+            fi
+          done
+
+          if [ "$HAS_ERRORS" -eq 1 ]; then
+            echo ""
+            echo "🚨 One or more Prometheus rule tests failed."
+            exit 1
+          else
+            echo ""
+            echo "🎉 All Prometheus rule tests passed."
+          fi
+      silent: false

config/default/kustomization.yaml

@@ -14,11 +14,13 @@ namePrefix: network-services-operator-
 #  pairs:
 #    someName: someValue
 
+components:
+- ../resource-metrics
+
 resources:
 - ../crd
 - ../rbac
 - ../manager
-- ../resource-metrics
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 - ../webhook
@@ -35,11 +37,11 @@ resources:
 #- ../network-policy
 
 # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager
-patches:
+#patches:
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
-- path: manager_webhook_patch.yaml
+#- path: manager_webhook_patch.yaml
 
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
 # Uncomment the following replacements to add the cert-manager CA injection annotations

config/resource-metrics/gateways.yaml

@@ -51,6 +51,17 @@ spec:
             labelsFromPath:
               type: ["type"]
             valueFrom: ["status"]
+      - name: "status_condition_last_transition_time"
+        help: "last transition time for status conditions"
+        each:
+          type: Gauge
+          gauge:
+            path: [status, conditions]
+            labelsFromPath:
+              type: ["type"]
+              reason: ["reason"]
+              status: ["status"]
+            valueFrom: ["lastTransitionTime"]
       - name: "custom_hostname"
         help: "Custom hostname defined on the gateway"
         errorLogV: 10

config/telemetry/alerts/gateways.yaml

@@ -0,0 +1,41 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: nso-slo
+  namespace: nso-slo
+spec:
+  groups:
+  - name: nso-slo
+    interval: 30s
+    rules:
+    - alert: GatewayNotReadySLOViolation
+      expr: |
+        sum(time() - datum_cloud_networking_gateway_created) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) > 60
+        unless (
+          sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) == 2
+        )
+      for: 0s
+      labels:
+        severity: critical
+        slo_violation: "true"
+      annotations:
+        summary: "Gateway {{ $labels.resource_name }} is taking longer than 60 seconds to reach Ready status"
+        description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in creation state for {{ $value }} seconds without reaching Ready status (Accepted=True AND Programmed=True), which exceeds the 60-second SLO threshold."
+
+    - alert: GatewayDegradedSLOViolation
+      expr: |
+        (
+          sum(time() - datum_cloud_networking_gateway_status_condition_last_transition_time{type=~"Accepted|Programmed", status="False"})
+          by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name)
+        ) > 60
+        and (
+          sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1)
+          by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) < 2
+        )
+      for: 0s
+      labels:
+        severity: critical
+        slo_violation: "true"
+      annotations:
+        summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
+        description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."

config/telemetry/alerts/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+  - gateways.yaml

test/prometheus-rules/gateways/nso-slo-rules.yaml

@@ -0,0 +1,35 @@
+groups:
+- name: nso-slo
+  interval: 30s
+  rules:
+  - alert: GatewayNotReadySLOViolation
+    expr: |
+      sum(time() - datum_cloud_networking_gateway_created) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) > 60
+      unless (
+        sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1) by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) == 2
+      )
+    for: 0s
+    labels:
+      severity: critical
+      slo_violation: "true"
+    annotations:
+      summary: "Gateway {{ $labels.resource_name }} is taking longer than 60 seconds to reach Ready status"
+      description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in creation state for {{ $value }} seconds without reaching Ready status (Accepted=True AND Programmed=True), which exceeds the 60-second SLO threshold."
+
+  - alert: GatewayDegradedSLOViolation
+    expr: |
+      (
+        sum(time() - datum_cloud_networking_gateway_status_condition_last_transition_time{type=~"Accepted|Programmed", status="False"})
+        by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name)
+      ) > 60
+      and (
+        sum(datum_cloud_networking_gateway_status{type=~"Accepted|Programmed"} == 1)
+        by (resourcemanager_datumapis_com_project_name, resource_namespace, resource_name) < 2
+      )
+    for: 0s
+    labels:
+      severity: critical
+      slo_violation: "true"
+    annotations:
+      summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
+      description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."

test/prometheus-rules/gateways/nso-slo-tests.yaml

@@ -0,0 +1,101 @@
+rule_files:
+  - nso-slo-rules.yaml
+
+evaluation_interval: 30s
+
+tests:
+  # Test for GatewayNotReadySLOViolation - Gateway created more than 60s ago but not ready
+  - interval: 1m
+    input_series:
+      # Gateway created 120 seconds ago (eval_time 60s - 120s = timestamp -60)
+      - series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway"}'
+        values: '-60+0x3'
+      # Only has Accepted=1 status, missing Programmed
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway", type="Accepted"}'
+        values: '1+0x3'
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="test-gateway", type="Programmed"}'
+        values: '0+0x3'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: GatewayNotReadySLOViolation
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              slo_violation: "true"
+              resourcemanager_datumapis_com_project_name: test-project
+              resource_namespace: test-ns
+              resource_name: test-gateway
+            exp_annotations:
+              summary: "Gateway test-gateway is taking longer than 60 seconds to reach Ready status"
+              description: "Gateway test-gateway in namespace test-ns has been in creation state for 120 seconds without reaching Ready status (Accepted=True AND Programmed=True), which exceeds the 60-second SLO threshold."
+
+  # Test for GatewayNotReadySLOViolation - Gateway ready (should NOT alert)
+  - interval: 1m
+    input_series:
+      # Gateway created 120 seconds ago
+      - series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway"}'
+        values: '-60+0x3'
+      # Both statuses are 1 (Ready)
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway", type="Accepted"}'
+        values: '1+0x3'
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="ready-gateway", type="Programmed"}'
+        values: '1+0x3'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: GatewayNotReadySLOViolation
+        exp_alerts: []
+
+  # Test for GatewayDegradedSLOViolation - Gateway degraded for more than 60s
+  - interval: 1m
+    input_series:
+      # Transition to False 120 seconds ago (eval_time 60s - 120s = timestamp -60)
+      - series: 'datum_cloud_networking_gateway_status_condition_last_transition_time{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Accepted", status="False"}'
+        values: '-60+0x3'
+      # Current status is False (0)
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Accepted"}'
+        values: '0+0x3'
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="degraded-gateway", type="Programmed"}'
+        values: '1+0x3'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: GatewayDegradedSLOViolation
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              slo_violation: "true"
+              resourcemanager_datumapis_com_project_name: test-project
+              resource_namespace: test-ns
+              resource_name: degraded-gateway
+            exp_annotations:
+              summary: "Gateway degraded-gateway has been degraded for over 60 seconds"
+              description: "Gateway degraded-gateway in namespace test-ns has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
+
+  # Test for GatewayDegradedSLOViolation - Gateway functional (should NOT alert)
+  - interval: 1m
+    input_series:
+      # Both statuses are 1 (functional)
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="healthy-gateway", type="Accepted"}'
+        values: '1+0x3'
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="healthy-gateway", type="Programmed"}'
+        values: '1+0x3'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: GatewayDegradedSLOViolation
+        exp_alerts: []
+
+  # Edge case test - Gateway created exactly 60s ago (should NOT alert)
+  - interval: 1m
+    input_series:
+      # Gateway created exactly 60 seconds ago (time() - created = 60)
+      # If eval_time is 1m (60s), then created should be 0
+      - series: 'datum_cloud_networking_gateway_created{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway"}'
+        values: '0+0x3'
+      # Only has Accepted
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway", type="Accepted"}'
+        values: '1+0x3'
+      - series: 'datum_cloud_networking_gateway_status{resourcemanager_datumapis_com_project_name="test-project", resource_namespace="test-ns", resource_name="edge-case-gateway", type="Programmed"}'
+        values: '0+0x3'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: GatewayNotReadySLOViolation
+        exp_alerts: []