[18.0-fr3] Add node failure tolerations to all service operators and openstackclient

bindata/operator/managers.yaml

@@ -85,6 +85,15 @@ spec:
         runAsNonRoot: true
       serviceAccountName: {{ .Name }}-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
 {{- if isEnvVarTrue .Deployment.Manager.Env "ENABLE_WEBHOOKS" }}
       volumes:
       - name: cert

bindata/operator/operator.yaml

@@ -133,6 +133,15 @@ spec:
         runAsNonRoot: true
       serviceAccountName: openstack-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - effect: NoExecute
+        key: node.kubernetes.io/not-ready
+        operator: Exists
+        tolerationSeconds: 120
+      - effect: NoExecute
+        key: node.kubernetes.io/unreachable
+        operator: Exists
+        tolerationSeconds: 120
       volumes:
       - name: cert
         secret:

bindata/operator/rabbit.yaml

@@ -46,3 +46,12 @@ spec:
             memory: {{ .RabbitmqOperator.Deployment.Manager.Resources.Requests.Memory }}
       serviceAccountName: rabbitmq-cluster-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120

config/manager/manager.yaml

@@ -72,3 +72,12 @@ spec:
             customRequests: replace_me #NOTE: this is used via the Makefile to inject a custom template that kustomize won't allow
       serviceAccountName: openstack-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120

config/operator/deployment/deployment.yaml

@@ -106,3 +106,12 @@ spec:
             memory: 128Mi
       serviceAccountName: openstack-operator-controller-operator
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120

config/operator/managers.yaml

@@ -85,6 +85,15 @@ spec:
         runAsNonRoot: true
       serviceAccountName: {{ .Name }}-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
 {{- if isEnvVarTrue .Deployment.Manager.Env "ENABLE_WEBHOOKS" }}
       volumes:
       - name: cert

config/operator/rabbit.yaml

@@ -46,3 +46,12 @@ spec:
             memory: {{ .RabbitmqOperator.Deployment.Manager.Resources.Requests.Memory }}
       serviceAccountName: rabbitmq-cluster-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120

controllers/client/openstackclient_controller.go

@@ -378,6 +378,18 @@ func (r *OpenStackClientReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		)
 	}
 
+	// if pod is stuck in terminating state for more than 3 minutes, force delete it
+	if osclient.DeletionTimestamp != nil {
+		terminatingDuration := time.Since(osclient.DeletionTimestamp.Time)
+		if terminatingDuration > time.Minute*3 {
+			// Force delete only truly stuck pods
+			err := r.Client.Delete(ctx, osclient, client.GracePeriodSeconds(0))
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("Failed to force delete pod: %w", err)
+			}
+		}
+	}
+
 	podReady := false
 
 	for _, condition := range osclient.Status.Conditions {

pkg/openstackclient/funcs.go

@@ -95,6 +95,20 @@ func ClientPodSpec(
 				VolumeMounts: volumeMounts,
 			},
 		},
+		Tolerations: []corev1.Toleration{
+			{
+				Key:               "node.kubernetes.io/not-ready",
+				Operator:          corev1.TolerationOpExists,
+				Effect:            corev1.TaintEffectNoExecute,
+				TolerationSeconds: &[]int64{120}[0],
+			},
+			{
+				Key:               "node.kubernetes.io/unreachable",
+				Operator:          corev1.TolerationOpExists,
+				Effect:            corev1.TaintEffectNoExecute,
+				TolerationSeconds: &[]int64{120}[0],
+			},
+		},
 	}
 
 	if instance.Spec.NodeSelector != nil {