Add kuttl tests for lightspeed-app-server /metrics endpoint

test/kuttl/common/openstack-lightspeed-instance/assert-metrics-endpoint.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      #!/bin/bash
+      set -euo pipefail
+
+      NAMESPACE="openstack-lightspeed"
+      CONTAINER="lightspeed-service-api"
+      METRICS_PORT=8443
+      METRICS_PATH="/metrics"
+      SA_NAME="lightspeed-metrics-test"
+
+      echo "Getting bearer token..."
+      TOKEN="$(oc create token "$SA_NAME" -n "$NAMESPACE")"
+
+      echo "Waiting for lightspeed-stack-deployment pod to be ready..."
+      oc wait --for=condition=Ready pod \
+        -l app.kubernetes.io/name=openstack-lightspeed-app-server \
+        -n "$NAMESPACE" \
+        --timeout=120s
+
+      echo "Getting pod name..."
+      POD_NAME="$(oc get pods \
+        -l app.kubernetes.io/name=openstack-lightspeed-app-server \
+        -n "$NAMESPACE" \
+        -o jsonpath='{.items[0].metadata.name}')"
+
+      if [ -z "$POD_NAME" ]; then
+        echo "ERROR: No pod found for lightspeed-app-server"
+        exit 1
+      fi
+
+      echo "Fetching metrics from pod $POD_NAME (container: $CONTAINER)..."
+      METRICS_OUTPUT="$(oc exec -n "$NAMESPACE" "$POD_NAME" \
+        -c "$CONTAINER" -- \
+        curl -sk -H "Authorization: Bearer ${TOKEN}" \
+        "https://localhost:${METRICS_PORT}${METRICS_PATH}")"
+
+      if [ -z "$METRICS_OUTPUT" ]; then
+        echo "ERROR: Empty response from /metrics endpoint"
+        exit 1
+      fi
+
+      echo "Validating response is valid Prometheus exposition format..."
+      if ! echo "$METRICS_OUTPUT" | grep -q "^# HELP "; then
+        echo "ERROR: Response does not contain Prometheus HELP declarations"
+        echo "Response (first 500 chars):"
+        echo "$METRICS_OUTPUT" | head -c 500
+        exit 1
+      fi
+
+      if ! echo "$METRICS_OUTPUT" | grep -q "^# TYPE "; then
+        echo "ERROR: Response does not contain Prometheus TYPE declarations"
+        echo "Response (first 500 chars):"
+        echo "$METRICS_OUTPUT" | head -c 500
+        exit 1
+      fi
+
+      if ! echo "$METRICS_OUTPUT" | grep -q "^# TYPE ls_llm_calls_total "; then
+        echo "ERROR: Missing expected lightspeed metric: ls_llm_calls_total"
+        echo "Response (first 500 chars):"
+        echo "$METRICS_OUTPUT" | head -c 500
+        exit 1
+      fi
+
+      echo "Metrics endpoint is reachable and returns valid Prometheus format"
+    timeout: 180

test/kuttl/common/openstack-lightspeed-instance/cleanup-openstack-lightspeed-instance.yaml

@@ -10,4 +10,14 @@ delete:
     kind: PersistentVolumeClaim
     name: openstack-lightspeed-database
     namespace: openstack-lightspeed
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRoleBinding
+    name: lightspeed-metrics-test-binding
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    name: lightspeed-metrics-test-role
+  - apiVersion: v1
+    kind: ServiceAccount
+    name: lightspeed-metrics-test
+    namespace: openstack-lightspeed
 

test/kuttl/common/openstack-lightspeed-instance/create-openstack-lightspeed-instance.yaml

@@ -19,3 +19,30 @@ spec:
     dataverseExporterLogLevel: DEBUG
   dev:
     okpChunkFilterQuery: "product:(*openstack* OR *openshift*)"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: lightspeed-metrics-test
+  namespace: openstack-lightspeed
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: lightspeed-metrics-test-role
+rules:
+- nonResourceURLs: ["/ls-access"]
+  verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: lightspeed-metrics-test-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: lightspeed-metrics-test-role
+subjects:
+- kind: ServiceAccount
+  name: lightspeed-metrics-test
+  namespace: openstack-lightspeed

test/kuttl/common/openstack-lightspeed-instance/errors-openstack-lightspeed-instance.yaml

@@ -114,3 +114,21 @@ kind: ConfigMap
 metadata:
   name: vector-db-scripts
   namespace: openstack-lightspeed
+
+# Metrics test RBAC resources should be gone
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: lightspeed-metrics-test-binding
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: lightspeed-metrics-test-role
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: lightspeed-metrics-test
+  namespace: openstack-lightspeed

test/kuttl/tests/basic-openstack-lightspeed-configuration/07-assert-metrics-endpoint.yaml

@@ -0,0 +1 @@
+../../common/openstack-lightspeed-instance/assert-metrics-endpoint.yaml