test: Shutdown Kafka Brokers before ZooKeeper/KRaft Controllers (#956)

* test: Add final step to scale down kafka

This ensures Kafka pods don't hang when zookeeper is shutdown before kafka on namespace deletion by kuttl

* test: Add final step to scale down kafka when using KRaft

This ensures Kafka brokers don't hang when kraft controlers are shutdown on namespace deletion by kuttl

* Revert "test: Add final step to scale down kafka when using KRaft"

This reverts commit cc853b9.

This relies on a bug fix that should come in with #955

* test(logging): Scale down the custom-log-config RoleGroup

Co-authored-by: Andrew Kenworthy <andrew.kenworthy@stackable.de>

* test(delete-role): Add note about not scaling down the already deleted RoleGroup

---------

Co-authored-by: Andrew Kenworthy <andrew.kenworthy@stackable.de>

Author: NickLarsenNZ

tests/templates/kuttl/cluster-operation/90-shutdown-kafka.yaml

@@ -0,0 +1,11 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously,
+# and Kafka hangs on controlled-shutdown ZooKeeper timeouts.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/delete-rolegroup/90-shutdown-kafka.yaml

@@ -0,0 +1,12 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously,
+# and Kafka hangs on controlled-shutdown ZooKeeper timeouts.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Note: By the time this script runs, the secondary RoleGroup has already been deleted, therefore we don't scale it down.
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/kerberos/90-shutdown-kafka.yaml

@@ -0,0 +1,11 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously,
+# and Kafka hangs on controlled-shutdown ZooKeeper timeouts.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/logging/90-shutdown-kafka.yaml

@@ -0,0 +1,11 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously,
+# and Kafka hangs on controlled-shutdown ZooKeeper timeouts.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"automatic-log-config":{"replicas":0}, "custom-log-config":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/opa/90-shutdown-kafka.yaml

@@ -0,0 +1,11 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously,
+# and Kafka hangs on controlled-shutdown ZooKeeper timeouts.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/smoke/90-shutdown-kafka.yaml

@@ -0,0 +1,11 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously,
+# and Kafka hangs on controlled-shutdown ZooKeeper timeouts.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s

tests/templates/kuttl/tls/90-shutdown-kafka.yaml

@@ -0,0 +1,11 @@
+---
+# Scale Kafka down before kuttl deletes the namespace.
+# Without this, ZooKeeper and Kafka are terminated simultaneously,
+# and Kafka hangs on controlled-shutdown ZooKeeper timeouts.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      kubectl patch kafkacluster test-kafka -n $NAMESPACE --type merge -p '{"spec":{"brokers":{"roleGroups":{"default":{"replicas":0}}}}}'
+  - script: |
+      kubectl wait --for=delete pod -l app.kubernetes.io/instance=test-kafka -n $NAMESPACE --timeout=300s