Apply offboarding taint once VMs are evicted

Gate on Evicting=False to avoid racing with live-migration. Watch
Hypervisor status changes (evictingConditionChangedPredicate) so the
taint is applied promptly when the condition transitions.

Author: fwiesel

internal/controller/constants.go

@@ -21,4 +21,9 @@ package controller
 const (
 	labelHypervisor   = "nova.openstack.cloud.sap/virt-driver"
 	testAggregateName = "tenant_filter_tests"
+
+	// taintKeyOffboarding is used as a NoExecute taint. nova-compute and
+	// neutron agent pods do not tolerate it (the kvm-node-agent and the
+	// signalling pod do), so applying it forces those agents off the node.
+	taintKeyOffboarding = "kvm.cloud.sap/offboarding"
 )

internal/controller/gardener_node_lifecycle_controller.go

@@ -33,9 +33,13 @@ import (
 	v1 "k8s.io/client-go/applyconfigurations/meta/v1"
 	policyv1ac "k8s.io/client-go/applyconfigurations/policy/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	logger "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	kvmv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
 )
@@ -85,6 +89,19 @@ func (r *GardenerNodeLifecycleController) Reconcile(ctx context.Context, req ctr
 		return ctrl.Result{}, nil
 	}
 
+	// Apply the offboarding taint once VMs are gone; gate on Evicting=False
+	// to avoid racing with live-migration.
+	if hv.Spec.Maintenance == kvmv1.MaintenanceTermination &&
+		meta.IsStatusConditionFalse(hv.Status.Conditions, kvmv1.ConditionTypeEvicting) {
+		patched, err := r.ensureOffboardingTaint(ctx, node)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to ensure offboarding taint: %w", err)
+		}
+		if patched {
+			return ctrl.Result{}, nil
+		}
+	}
+
 	// We do not care about the particular value, as long as it isn't an error
 	var minAvailable int32 = 1
 
@@ -115,6 +132,30 @@ func (r *GardenerNodeLifecycleController) Reconcile(ctx context.Context, req ctr
 	return ctrl.Result{}, nil
 }
 
+// ensureOffboardingTaint adds the offboarding NoExecute taint if not already
+// present. Returns true when a patch was issued (caller should return early).
+func (r *GardenerNodeLifecycleController) ensureOffboardingTaint(ctx context.Context, node *corev1.Node) (bool, error) {
+	for _, t := range node.Spec.Taints {
+		if t.Key == taintKeyOffboarding && t.Effect == corev1.TaintEffectNoExecute {
+			return false, nil
+		}
+	}
+
+	log := logger.FromContext(ctx)
+	log.Info("Adding offboarding taint to node",
+		"node", node.Name,
+		"taint", taintKeyOffboarding,
+		"effect", corev1.TaintEffectNoExecute)
+
+	// StrategicMergeFrom merges taints by key, preserving concurrent additions.
+	patch := k8sclient.StrategicMergeFrom(node.DeepCopy())
+	node.Spec.Taints = append(node.Spec.Taints, corev1.Taint{
+		Key:    taintKeyOffboarding,
+		Effect: corev1.TaintEffectNoExecute,
+	})
+	return true, r.Patch(ctx, node, patch, k8sclient.FieldOwner(MaintenanceControllerName))
+}
+
 func (r *GardenerNodeLifecycleController) ensureBlockingPodDisruptionBudget(ctx context.Context, node *corev1.Node, minAvailable int32) error {
 	name := nameForNode(node)
 	nodeLabels := labelsForNode(node)
@@ -226,10 +267,48 @@ func (r *GardenerNodeLifecycleController) SetupWithManager(mgr ctrl.Manager, nam
 	_ = logger.FromContext(ctx)
 	r.namespace = namespace
 
+	hypervisorToNode := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &corev1.Node{})
+
+	// Maintenance=termination bumps generation; Evicting status changes do not.
+	hypervisorRelevantChange := predicate.Or(
+		predicate.GenerationChangedPredicate{},
+		evictingConditionChangedPredicate{},
+	)
+
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(MaintenanceControllerName).
 		For(&corev1.Node{}).
-		Owns(&appsv1.Deployment{}). // trigger the r.Reconcile whenever an Own-ed deployment is created/updated/deleted
+		Watches(&kvmv1.Hypervisor{}, hypervisorToNode,
+			builder.WithPredicates(hypervisorRelevantChange),
+		).
+		Owns(&appsv1.Deployment{}).
 		Owns(&policyv1.PodDisruptionBudget{}).
 		Complete(r)
 }
+
+// evictingConditionChangedPredicate complements GenerationChangedPredicate,
+// which ignores status-only updates.
+type evictingConditionChangedPredicate struct {
+	predicate.Funcs
+}
+
+func (evictingConditionChangedPredicate) Update(e event.UpdateEvent) bool {
+	if e.ObjectOld == nil || e.ObjectNew == nil {
+		return false
+	}
+	oldHv, ok1 := e.ObjectOld.(*kvmv1.Hypervisor)
+	newHv, ok2 := e.ObjectNew.(*kvmv1.Hypervisor)
+	if !ok1 || !ok2 {
+		return false
+	}
+	oldCond := meta.FindStatusCondition(oldHv.Status.Conditions, kvmv1.ConditionTypeEvicting)
+	newCond := meta.FindStatusCondition(newHv.Status.Conditions, kvmv1.ConditionTypeEvicting)
+	switch {
+	case oldCond == nil && newCond == nil:
+		return false
+	case oldCond == nil || newCond == nil:
+		return true
+	default:
+		return oldCond.Status != newCond.Status
+	}
+}

internal/controller/gardener_node_lifecycle_controller_test.go

@@ -207,4 +207,63 @@ var _ = Describe("Gardener Maintenance Controller", func() {
 			Expect(pdb.Spec.MinAvailable).To(HaveField("IntVal", BeNumerically("==", int32(0))))
 		})
 	})
+
+	Context("Offboarding taint", func() {
+		findOffboardingTaint := func(node *corev1.Node) *corev1.Taint {
+			for i := range node.Spec.Taints {
+				t := &node.Spec.Taints[i]
+				if t.Key == taintKeyOffboarding && t.Effect == corev1.TaintEffectNoExecute {
+					return t
+				}
+			}
+			return nil
+		}
+
+		When("the hypervisor is in maintenance termination and the VMs have been evicted", func() {
+			BeforeEach(func(ctx SpecContext) {
+				hypervisor := &kvmv1.Hypervisor{}
+				Expect(k8sClient.Get(ctx, name, hypervisor)).To(Succeed())
+				hypervisor.Spec.Maintenance = kvmv1.MaintenanceTermination
+				Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed())
+
+				meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{
+					Type:    kvmv1.ConditionTypeEvicting,
+					Status:  metav1.ConditionFalse,
+					Reason:  "Succeeded",
+					Message: "All VMs evicted",
+				})
+				Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed())
+			})
+
+			It("should add the offboarding NoExecute taint to the node", func(ctx SpecContext) {
+				_, err := controller.Reconcile(ctx, reconcileReq)
+				Expect(err).NotTo(HaveOccurred())
+
+				node := &corev1.Node{}
+				Expect(k8sClient.Get(ctx, name, node)).To(Succeed())
+				taint := findOffboardingTaint(node)
+				Expect(taint).NotTo(BeNil())
+				Expect(taint.Key).To(Equal(taintKeyOffboarding))
+				Expect(taint.Effect).To(Equal(corev1.TaintEffectNoExecute))
+			})
+
+			It("should be idempotent: a second reconcile must not add a duplicate taint", func(ctx SpecContext) {
+				_, err := controller.Reconcile(ctx, reconcileReq)
+				Expect(err).NotTo(HaveOccurred())
+				_, err = controller.Reconcile(ctx, reconcileReq)
+				Expect(err).NotTo(HaveOccurred())
+
+				node := &corev1.Node{}
+				Expect(k8sClient.Get(ctx, name, node)).To(Succeed())
+
+				count := 0
+				for _, t := range node.Spec.Taints {
+					if t.Key == taintKeyOffboarding {
+						count++
+					}
+				}
+				Expect(count).To(Equal(1))
+			})
+		})
+	})
 })