test: Add final step to scale down kafka when using KRaft

This ensures Kafka brokers don't hang when kraft controlers are shutdown on namespace deletion by kuttl

Author: NickLarsenNZ

tests/templates/kuttl/configuration/90-shutdown-kafka.yaml

@@ -0,0 +1,16 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, Kafka pods may hang during shutdown.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Scale down brokers first, since they depend on controllers.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=broker -n $NAMESPACE --timeout=300s
+  # Then scale down controllers once all brokers are gone.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"controllers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/operations-kraft/90-shutdown-kafka.yaml

@@ -0,0 +1,16 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, Kafka pods may hang during shutdown.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Scale down brokers first, since they depend on controllers.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=broker -n $NAMESPACE --timeout=300s
+  # Then scale down controllers once all brokers are gone.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"controllers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/smoke-kraft/90-shutdown-kafka.yaml

@@ -0,0 +1,16 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, Kafka pods may hang during shutdown.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Scale down brokers first, since they depend on controllers.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0},"automatic-log-config":{"replicas":0},"custom-log-config":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=broker -n $NAMESPACE --timeout=300s
+  # Then scale down controllers once all brokers are gone.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"controllers":{"roleGroups":{"automatic-log-config":{"replicas":0},"custom-log-config":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/upgrade/90-shutdown-kafka.yaml

@@ -0,0 +1,16 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, Kafka pods may hang during shutdown.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Scale down brokers first, since they depend on controllers.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka,app.kubernetes.io/component=broker -n $NAMESPACE --timeout=300s
+  # Then scale down controllers once all brokers are gone.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"controllers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s