chore: Reduce severity of Pod eviciton errors

Author: sbernauer

Cargo.lock

@@ -16,6 +16,7 @@ anyhow = "1.0"
 built = { version = "0.8", features = ["chrono", "git2"] }
 clap = "4.5"
 futures = { version = "0.3", features = ["compat"] }
+http = "1.3"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 snafu = "0.8"

Cargo.nix

@@ -13,6 +13,7 @@ stackable-operator.workspace = true
 
 anyhow.workspace = true
 clap.workspace = true
+http.workspace = true
 futures.workspace = true
 serde.workspace = true
 serde_json.workspace = true

Cargo.toml

@@ -1,6 +1,7 @@
 use std::{sync::Arc, time::Duration};
 
 use futures::StreamExt;
+use http::StatusCode;
 use snafu::{OptionExt, ResultExt, Snafu};
 use stackable_operator::{
     client::Client,
@@ -11,10 +12,10 @@ use stackable_operator::{
     kube::{
         self,
         api::{EvictParams, PartialObjectMeta},
-        core::DynamicObject,
+        core::{DynamicObject, ErrorResponse},
         runtime::{
             Controller,
-            controller::Action,
+            controller::{self, Action},
             events::{Recorder, Reporter},
             reflector::ObjectRef,
             watcher,
@@ -96,10 +97,7 @@ pub async fn start(client: &Client, watch_namespace: &WatchNamespace) {
                 // The event_recorder needs to be shared across all invocations, so that
                 // events are correctly aggregated
                 let event_recorder = event_recorder.clone();
-                async move {
-                    report_controller_reconciled(&event_recorder, FULL_CONTROLLER_NAME, &result)
-                        .await;
-                }
+                async move { report_result(result, event_recorder).await }
             },
         )
         .await;
@@ -192,6 +190,45 @@ async fn reconcile(pod: Arc<PartialObjectMeta<Pod>>, ctx: Arc<Ctx>) -> Result<Ac
     }
 }
 
+/// Reports the result of reconciliation.
+///
+/// The Pod restart controller has special handling, as it produced lot's of error messages below.
+/// They are expected, as we intentionally use the `Evict` API to restart Pods before e.g. the
+/// certificate expires. We roll out PDBs by default. If we try to restart multiple Pods that are
+/// part of a PDB, we get this errors.
+/// Because of this, we don't emit an error for this case, but only product a INFO trace.
+///
+/// `ERROR stackable_operator::logging::controller: Failed to reconcile object controller.name="pod.restarter.commons.stackable.tech" error=reconciler for object Pod.v1./trino-worker-default-0.default failed error.sources=[failed to evict Pod, ApiError: Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests (ErrorResponse { status: "Failure", message: "Cannot evict pod as it would violate the pod's disruption budget.", reason: "TooManyRequests", code: 429 }), Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests]`
+async fn report_result(
+    result: Result<
+        (ObjectRef<PartialObjectMeta<Pod>>, Action),
+        controller::Error<Error, watcher::Error>,
+    >,
+    event_recorder: Arc<Recorder>,
+) {
+    if let Err(controller::Error::ReconcilerFailed(
+        Error::EvictPod {
+            source: evict_pod_error,
+        },
+        _,
+    )) = &result
+    {
+        const TOO_MANY_REQUESTS_HTTP_CODE: u16 = StatusCode::TOO_MANY_REQUESTS.as_u16();
+        if let kube::Error::Api(ErrorResponse {
+            code: TOO_MANY_REQUESTS_HTTP_CODE,
+            ..
+        }) = evict_pod_error
+        {
+            tracing::info!(
+                ?evict_pod_error,
+                "Tried to evict Pod, but wasn't allowed to do so, as it would violate the Pod's disruption budget. Retrying later"
+            );
+        }
+    }
+
+    report_controller_reconciled(&event_recorder, FULL_CONTROLLER_NAME, &result).await;
+}
+
 fn error_policy(_obj: Arc<PartialObjectMeta<Pod>>, _error: &Error, _ctx: Arc<Ctx>) -> Action {
     Action::requeue(Duration::from_secs(5))
 }