Add kuttl test for routing

Author: sbernauer

tests/templates/kuttl/smoke/10-assert.yaml

@@ -3,4 +3,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 720
 commands:
-  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-default-1 --timeout 719s
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-s-1 --timeout 719s
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-m-1 --timeout 719s

…tes/kuttl/smoke/10-install-trino.yaml.j2 …es/kuttl/smoke/10-install-trinos.yaml.j2tests/templates/kuttl/smoke/10-install-trino.yaml.j2 renamed to tests/templates/kuttl/smoke/10-install-trinos.yaml.j2 tests/templates/kuttl/smoke/10-install-trino.yaml.j2 renamed to tests/templates/kuttl/smoke/10-install-trinos.yaml.j2

@@ -1,8 +1,9 @@
+{% for name in ['trino-s-1', 'trino-m-1'] %}
 ---
 apiVersion: trino.stackable.tech/v1alpha1
 kind: TrinoCluster
 metadata:
-  name: trino-default-1
+  name: {{ name }}
 spec:
   image:
 {% if test_scenario['values']['trino'].find(",") > 0 %}
@@ -23,19 +24,32 @@ spec:
 {% endif %}
   coordinators:
     config:
+      resources:
+        cpu:
+          min: 250m
+          max: "1"
+        memory:
+          limit: 2Gi
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
     roleGroups:
       default:
         replicas: 1
   workers:
     config:
+      resources:
+        cpu:
+          min: 250m
+          max: "1"
+        memory:
+          limit: 3Gi
       gracefulShutdownTimeout: 60s # Let the test run faster
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
     roleGroups:
       default:
         replicas: 1
+{% endfor %}
 ---
 apiVersion: authentication.stackable.tech/v1alpha1
 kind: AuthenticationClass

tests/templates/kuttl/smoke/20-install-trino-lb.j2

@@ -9,6 +9,9 @@ kind: Deployment
 metadata:
   name: trino-lb
 spec:
+  # I'm a bit surprised that 3 replicas work in combination with inMemory persistence :)
+  # Maybe Kubernetes is sticky for some reason, maybe trino-clients will retry call until they get
+  # to the correct trino-lb instance...
   replicas: 3
   selector:
     matchLabels:

tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2

@@ -10,8 +10,8 @@ trinoClusterGroups:
   default:
     maxRunningQueries: 1
     trinoClusters:
-      - name: trino-default-1
-        endpoint: https://trino-default-1-coordinator:8443
+      - name: trino-s-1
+        endpoint: https://trino-s-1-coordinator:8443
         credentials:
           username: admin
           password: adminadmin

tests/templates/kuttl/smoke/30-assert.yaml

@@ -2,6 +2,6 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: send-queries
+  name: test-queries
 status:
   succeeded: 1

tests/templates/kuttl/smoke/30-send-queries.yaml.j2

@@ -0,0 +1,54 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-queries
+spec:
+  template:
+    spec:
+      containers:
+        - name: test-queries
+          image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev
+          command:
+            - /bin/bash
+            - -x
+            - -euo
+            - pipefail
+            - -c
+            - |
+              # Query Trinos and trino-lb
+              COORDINATORS=(
+                  "https://trino-s-1-coordinator:8443"
+                  "https://trino-m-1-coordinator:8443"
+                  "https://trino-lb:8443"
+              )
+
+              export TRINO_USER="alice"
+              export TRINO_PASSWORD="alicealice"
+              QUERY="select count(*) from tpch.sf1.customer"
+
+              for COORDINATOR in "${COORDINATORS[@]}"; do
+                  echo "$QUERY" | java -jar trino-cli-executable.jar --server $COORDINATOR --insecure --user $TRINO_USER --password
+              done
+
+              # Send multiple queries in parallel to trino-lb
+              NUM_REQUESTS=10
+              TRINO_LB_ADDRESS="https://trino-lb:8443"
+
+              pids=()
+
+              for ((i = 1; i <= NUM_REQUESTS; i++)); do
+                  echo "$QUERY" | java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user $TRINO_USER --password &
+                  pids+=("$!")
+              done
+
+              # Wait for all processes to complete and check exit codes
+              for pid in "${pids[@]}"; do
+                  if ! wait "$pid"; then
+                      echo "One of the requests failed with a non-zero exit code."
+                      exit 1
+                  fi
+              done
+
+              echo "All queries completed successfully."
+      restartPolicy: OnFailure

tests/templates/kuttl/smoke/30-test-queries.yaml.j2

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl -n $NAMESPACE delete secret trino-lb-config || true
+  - script: kubectl -n $NAMESPACE create secret generic trino-lb-config --from-file=trino-lb-config.yaml=40_trino-lb-config.yaml

tests/templates/kuttl/smoke/40-change-routing.j2

@@ -0,0 +1,52 @@
+trinoLb:
+  externalAddress: https://trino-lb:8443
+  tls:
+    enabled: true
+    certPemFile: /certificates/cert.pem
+    keyPemFile: /certificates/key.pem
+  persistence:
+    inMemory: {}
+trinoClusterGroups:
+  s:
+    maxRunningQueries: 1
+    trinoClusters:
+      - name: trino-s-1
+        endpoint: https://trino-s-1-coordinator:8443
+        credentials:
+          username: admin
+          password: adminadmin
+  m:
+    maxRunningQueries: 1
+    trinoClusters:
+      - name: trino-m-1
+        endpoint: https://trino-m-1-coordinator:8443
+        credentials:
+          username: admin
+          password: adminadmin
+trinoClusterGroupsIgnoreCert: true
+
+routers:
+  - trinoRoutingGroupHeader:
+      headerName: X-Trino-Routing-Group
+  - clientTags:
+      oneOf: ["s"]
+      trinoClusterGroup: s
+  - clientTags:
+      oneOf: ["m"]
+      trinoClusterGroup: m
+  - pythonScript:
+      script: |
+        from typing import Optional
+
+        def targetClusterGroup(query: str, headers: dict[str, str]) -> Optional[str]:
+            user = get_user(headers)
+            if user == "alice":
+                return "s"
+            elif user == "bob":
+                return "m"
+            else:
+                return None
+
+        def get_user(headers: dict[str, str]) -> Optional[str]:
+            return headers.get("x-trino-user")
+routingFallback: s

tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2

@@ -0,0 +1,7 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl -n $NAMESPACE rollout restart deployment trino-lb
+  - script: kubectl -n $NAMESPACE rollout status deployment trino-lb --watch
+  - script: sleep 2