fix: Add orderly shutdown steps to all kuttl tests to prevent namespace deletion timeouts

During namespace deletion, ZooKeeper and Kafka are terminated
simultaneously. Kafka's controlled-shutdown retries ZK connections
indefinitely, keeping the process alive for the full grace period and
blocking namespace deletion past kuttl's 300s timeout.

For ZK-mode tests: scale brokers to 0 via the CRD so the operator
performs an orderly shutdown before ZK is removed.

For KRaft-mode tests: scale brokers to 0, delete the KafkaCluster CR
to stop reconciliation, then force-delete controller pods. Controllers
cannot be scaled via the CRD due to a "no Kraft controllers found to
build ConfigMap" error.

All tests also set gracefulShutdownTimeout: 60s to bound the worst-case
wait. Validated with a full nightly suite run (26/26 PASS).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: adwk67

tests/templates/kuttl/cluster-operation/20-install-kafka.yaml.j2

@@ -23,6 +23,7 @@ spec:
     zookeeperConfigMapName: test-zk
   brokers:
     config:
+      gracefulShutdownTimeout: 60s
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
     roleGroups:

tests/templates/kuttl/cluster-operation/90-shutdown-kafka.yaml

@@ -0,0 +1,18 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously during
+# namespace deletion. Kafka's controlled-shutdown retries ZK connections
+# indefinitely, keeping the process alive for the full grace period
+# and blocking namespace deletion well past kuttl's 300s timeout.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 600
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      if kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=120s 2>/dev/null; then
+        exit 0
+      fi
+      kubectl delete pods -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/configuration/10-install-kafka.yaml.j2

@@ -23,6 +23,7 @@ spec:
 {% endif %}
   controllers:
     config:
+      gracefulShutdownTimeout: 60s
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
@@ -57,6 +58,7 @@ spec:
         replicas: 1
   brokers:
     config:
+      gracefulShutdownTimeout: 60s
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:

tests/templates/kuttl/configuration/90-shutdown-kafka.yaml

@@ -0,0 +1,25 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Brokers are scaled via the CRD so the operator performs an orderly shutdown.
+# Once brokers are gone, we delete the KafkaCluster CR to stop the operator
+# reconciling, then force-delete any remaining controller pods. We cannot scale
+# controllers via the CRD because the operator errors with "no Kraft controllers
+# found to build ConfigMap", and scaling the StatefulSet directly is immediately
+# reversed by the operator's reconciliation loop.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 600
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      if kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=broker -n $NAMESPACE --timeout=120s 2>/dev/null; then
+        exit 0
+      fi
+      kubectl delete pods -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=broker -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=broker -n $NAMESPACE --timeout=300s
+  - script: |
+      kubectl delete kafkacluster test-kafka -n $NAMESPACE --wait=false 2>/dev/null || true
+  - script: |
+      kubectl delete pods -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=controller -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=controller -n $NAMESPACE --timeout=120s 2>/dev/null || true

tests/templates/kuttl/delete-rolegroup/90-shutdown-kafka.yaml

@@ -0,0 +1,18 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously during
+# namespace deletion. Kafka's controlled-shutdown retries ZK connections
+# indefinitely, keeping the process alive for the full grace period
+# and blocking namespace deletion well past kuttl's 300s timeout.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 600
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0},"secondary":{"replicas":0}}}}}'
+  - script: |
+      if kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=120s 2>/dev/null; then
+        exit 0
+      fi
+      kubectl delete pods -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/kerberos/90-shutdown-kafka.yaml

@@ -0,0 +1,18 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously during
+# namespace deletion. Kafka's controlled-shutdown retries ZK connections
+# indefinitely, keeping the process alive for the full grace period
+# and blocking namespace deletion well past kuttl's 300s timeout.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 600
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      if kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=120s 2>/dev/null; then
+        exit 0
+      fi
+      kubectl delete pods -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/logging/04-install-kafka.yaml.j2

@@ -52,6 +52,8 @@ spec:
     vectorAggregatorConfigMapName: kafka-vector-aggregator-discovery
     zookeeperConfigMapName: test-kafka-znode
   brokers:
+    config:
+      gracefulShutdownTimeout: 60s
     roleGroups:
       automatic-log-config:
         replicas: 1

tests/templates/kuttl/logging/90-shutdown-kafka.yaml

@@ -0,0 +1,18 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously during
+# namespace deletion. Kafka's controlled-shutdown retries ZK connections
+# indefinitely, keeping the process alive for the full grace period
+# and blocking namespace deletion well past kuttl's 300s timeout.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 600
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"automatic-log-config":{"replicas":0},"custom-log-config":{"replicas":0}}}}}'
+  - script: |
+      if kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=120s 2>/dev/null; then
+        exit 0
+      fi
+      kubectl delete pods -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/opa/30-install-kafka.yaml.j2

@@ -42,6 +42,7 @@ commands:
           config:
             logging:
               enableVectorAgent: true
+            gracefulShutdownTimeout: 60s
           roleGroups:
             default:
               replicas: 3

tests/templates/kuttl/opa/90-shutdown-kafka.yaml

@@ -0,0 +1,18 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously during
+# namespace deletion. Kafka's controlled-shutdown retries ZK connections
+# indefinitely, keeping the process alive for the full grace period
+# and blocking namespace deletion well past kuttl's 300s timeout.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 600
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      if kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=120s 2>/dev/null; then
+        exit 0
+      fi
+      kubectl delete pods -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s