add pre-stop sleep hook to controller kafka container

Author: maltesander

rust/operator-binary/src/resource/statefulset.rs

@@ -23,8 +23,8 @@ use stackable_operator::{
             apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy},
             core::v1::{
                 ConfigMapKeySelector, ConfigMapVolumeSource, ContainerPort, EnvVar, EnvVarSource,
-                ExecAction, ObjectFieldSelector, PodSpec, Probe, ServiceAccount, TCPSocketAction,
-                Volume,
+                ExecAction, Lifecycle, LifecycleHandler, ObjectFieldSelector, PodSpec, Probe,
+                ServiceAccount, SleepAction, TCPSocketAction, Volume,
             },
         },
         apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString},
@@ -756,10 +756,24 @@ pub fn build_controller_rolegroup_statefulset(
         )
         .context(AddVolumesAndVolumeMountsSnafu)?;
 
+    // Currently, Controllers shutdown very fast, too fast in most times (flakyness) for the Brokers
+    // to off load properly. The Brokers then try to connect to any controllers until the
+    // `gracefulShutdownTimeout` is reached and the pod is finally killed.
+    // The `pre-stop` hook will delay the kill signal to the Controllers to provide the Brokers more
+    // time to offload data.
+    let mut kafka_container = cb_kafka.build();
+    kafka_container.lifecycle = Some(Lifecycle {
+        pre_stop: Some(LifecycleHandler {
+            sleep: Some(SleepAction { seconds: 10 }),
+            ..Default::default()
+        }),
+        ..Default::default()
+    });
+
     pod_builder
         .metadata(metadata)
         .image_pull_secrets_from_product_image(resolved_product_image)
-        .add_container(cb_kafka.build())
+        .add_container(kafka_container)
         .affinity(&merged_config.affinity)
         .add_volume(Volume {
             name: "config".to_string(),