util: tools to drain and uncordon cluster

pkg/controllers/clusterresourceplacementeviction/controller.go

@@ -29,6 +29,7 @@ import (
 	"go.goms.io/fleet/pkg/utils/controller"
 	"go.goms.io/fleet/pkg/utils/controller/metrics"
 	"go.goms.io/fleet/pkg/utils/defaulter"
+	evictionutils "go.goms.io/fleet/pkg/utils/eviction"
 )
 
 // Reconciler reconciles a ClusterResourcePlacementEviction object.
@@ -59,7 +60,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
 		return runtime.Result{}, client.IgnoreNotFound(err)
 	}
 
-	if isEvictionInTerminalState(&eviction) {
+	if evictionutils.IsEvictionInTerminalState(&eviction) {
 		return runtime.Result{}, nil
 	}
 
@@ -189,7 +190,7 @@ func (r *Reconciler) executeEviction(ctx context.Context, validationResult *evic
 		return nil
 	}
 
-	if !isPlacementPresent(evictionTargetBinding) {
+	if !evictionutils.IsPlacementPresent(evictionTargetBinding) {
 		klog.V(2).InfoS("No resources have been placed for ClusterResourceBinding in target cluster",
 			"clusterResourcePlacementEviction", eviction.Name, "clusterResourceBinding", evictionTargetBinding.Name, "targetCluster", eviction.Spec.ClusterName)
 		markEvictionNotExecuted(eviction, condition.EvictionBlockedMissingPlacementMessage)
@@ -240,37 +241,6 @@ func (r *Reconciler) executeEviction(ctx context.Context, validationResult *evic
 	return nil
 }
 
-// isEvictionInTerminalState checks to see if eviction is in a terminal state.
-func isEvictionInTerminalState(eviction *placementv1beta1.ClusterResourcePlacementEviction) bool {
-	validCondition := eviction.GetCondition(string(placementv1beta1.PlacementEvictionConditionTypeValid))
-	if condition.IsConditionStatusFalse(validCondition, eviction.GetGeneration()) {
-		klog.V(2).InfoS("Invalid eviction, no need to reconcile", "clusterResourcePlacementEviction", eviction.Name)
-		return true
-	}
-
-	executedCondition := eviction.GetCondition(string(placementv1beta1.PlacementEvictionConditionTypeExecuted))
-	if executedCondition != nil {
-		klog.V(2).InfoS("Eviction has executed condition specified, no need to reconcile", "clusterResourcePlacementEviction", eviction.Name)
-		return true
-	}
-	return false
-}
-
-// isPlacementPresent checks to see if placement on target cluster could be present.
-func isPlacementPresent(binding *placementv1beta1.ClusterResourceBinding) bool {
-	if binding.Spec.State == placementv1beta1.BindingStateBound {
-		return true
-	}
-	if binding.Spec.State == placementv1beta1.BindingStateUnscheduled {
-		currentAnnotation := binding.GetAnnotations()
-		previousState, exist := currentAnnotation[placementv1beta1.PreviousBindingStateAnnotation]
-		if exist && placementv1beta1.BindingState(previousState) == placementv1beta1.BindingStateBound {
-			return true
-		}
-	}
-	return false
-}
-
 // isEvictionAllowed calculates if eviction allowed based on available bindings and spec specified in placement disruption budget.
 func isEvictionAllowed(bindings []placementv1beta1.ClusterResourceBinding, crp placementv1beta1.ClusterResourcePlacement, db placementv1beta1.ClusterResourcePlacementDisruptionBudget) (bool, int) {
 	availableBindings := 0

pkg/utils/eviction/eviction.go

@@ -0,0 +1,42 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+
+package eviction
+
+import (
+	"k8s.io/klog/v2"
+
+	placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
+	"go.goms.io/fleet/pkg/utils/condition"
+)
+
+// IsEvictionInTerminalState checks to see if eviction is in a terminal state.
+func IsEvictionInTerminalState(eviction *placementv1beta1.ClusterResourcePlacementEviction) bool {
+	if validCondition := eviction.GetCondition(string(placementv1beta1.PlacementEvictionConditionTypeValid)); condition.IsConditionStatusFalse(validCondition, eviction.GetGeneration()) {
+		klog.V(2).InfoS("Invalid eviction, no need to reconcile", "clusterResourcePlacementEviction", eviction.Name)
+		return true
+	}
+
+	if executedCondition := eviction.GetCondition(string(placementv1beta1.PlacementEvictionConditionTypeExecuted)); executedCondition != nil {
+		klog.V(2).InfoS("Eviction has executed condition specified, no need to reconcile", "clusterResourcePlacementEviction", eviction.Name)
+		return true
+	}
+	return false
+}
+
+// IsPlacementPresent checks to see if placement on target cluster could be present.
+func IsPlacementPresent(binding *placementv1beta1.ClusterResourceBinding) bool {
+	if binding.Spec.State == placementv1beta1.BindingStateBound {
+		return true
+	}
+	if binding.Spec.State == placementv1beta1.BindingStateUnscheduled {
+		currentAnnotation := binding.GetAnnotations()
+		previousState, exist := currentAnnotation[placementv1beta1.PreviousBindingStateAnnotation]
+		if exist && placementv1beta1.BindingState(previousState) == placementv1beta1.BindingStateBound {
+			return true
+		}
+	}
+	return false
+}

pkg/utils/eviction/eviction_test.go

@@ -0,0 +1,153 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+
+package eviction
+
+import (
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
+)
+
+func TestIsEvictionInTerminalState(t *testing.T) {
+	tests := []struct {
+		name     string
+		eviction *placementv1beta1.ClusterResourcePlacementEviction
+		want     bool
+	}{
+		{
+			name: "Invalid eviction - terminal state",
+			eviction: &placementv1beta1.ClusterResourcePlacementEviction{
+				Status: placementv1beta1.PlacementEvictionStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   string(placementv1beta1.PlacementEvictionConditionTypeValid),
+							Status: metav1.ConditionFalse,
+						},
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "Executed eviction set to true - terminal state",
+			eviction: &placementv1beta1.ClusterResourcePlacementEviction{
+				Status: placementv1beta1.PlacementEvictionStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   string(placementv1beta1.PlacementEvictionConditionTypeValid),
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   string(placementv1beta1.PlacementEvictionConditionTypeExecuted),
+							Status: metav1.ConditionTrue,
+						},
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "Executed eviction set to false - terminal state",
+			eviction: &placementv1beta1.ClusterResourcePlacementEviction{
+				Status: placementv1beta1.PlacementEvictionStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   string(placementv1beta1.PlacementEvictionConditionTypeValid),
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   string(placementv1beta1.PlacementEvictionConditionTypeExecuted),
+							Status: metav1.ConditionFalse,
+						},
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "Eviction with only valid condition set to true - non terminal state",
+			eviction: &placementv1beta1.ClusterResourcePlacementEviction{
+				Status: placementv1beta1.PlacementEvictionStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   string(placementv1beta1.PlacementEvictionConditionTypeValid),
+							Status: metav1.ConditionTrue,
+						},
+					},
+				},
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsEvictionInTerminalState(tt.eviction); got != tt.want {
+				t.Errorf("IsEvictionInTerminalState test failed got = %v, want = %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsPlacementPresent(t *testing.T) {
+	tests := []struct {
+		name    string
+		binding *placementv1beta1.ClusterResourceBinding
+		want    bool
+	}{
+		{
+			name: "Bound binding - placement present",
+			binding: &placementv1beta1.ClusterResourceBinding{
+				Spec: placementv1beta1.ResourceBindingSpec{
+					State: placementv1beta1.BindingStateBound,
+				},
+			},
+			want: true,
+		},
+		{
+			name: "Unscheduled binding with previous state bound - placement present",
+			binding: &placementv1beta1.ClusterResourceBinding{
+				Spec: placementv1beta1.ResourceBindingSpec{
+					State: placementv1beta1.BindingStateUnscheduled,
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						placementv1beta1.PreviousBindingStateAnnotation: string(placementv1beta1.BindingStateBound),
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "Unscheduled binding with no previous state - placement not present",
+			binding: &placementv1beta1.ClusterResourceBinding{
+				Spec: placementv1beta1.ResourceBindingSpec{
+					State: placementv1beta1.BindingStateUnscheduled,
+				},
+			},
+			want: false,
+		},
+		{
+			name: "Scheduled binding - placement not present",
+			binding: &placementv1beta1.ClusterResourceBinding{
+				Spec: placementv1beta1.ResourceBindingSpec{
+					State: placementv1beta1.BindingStateScheduled,
+				},
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsPlacementPresent(tt.binding); got != tt.want {
+				t.Errorf("IsPlacementPresent test failed got = %v, want = %v", got, tt.want)
+			}
+		})
+	}
+}

tools/draincluster/README.md

@@ -0,0 +1,77 @@
+# Steps to build draincluster as a kubectl plugin
+
+1. Build the binary for the `draincluster` tool by running the following command in the root directory of the fleet repo:
+
+```bash
+go build -o ./hack/tools/bin/kubectl-draincluster ./tools/draincluster/main.go
+```
+
+2. Copy the binary to a directory in your `PATH` so that it can be run as a kubectl plugin. For example, you can move it to
+`/usr/local/bin`:
+
+```bash
+sudo cp ./hack/tools/bin/kubectl-draincluster /usr/local/bin/
+```
+
+3. Make the binary executable by running the following command:
+
+```bash
+chmod +x /usr/local/bin/kubectl-draincluster
+```
+
+4. Verify that the plugin is recognized by kubectl by running the following command:
+
+```bash
+kubectl plugin list
+```
+
+you should see the `draincluster` plugin listed in the output:
+
+```
+The following compatible plugins are available:
+
+/usr/local/bin/kubectl-draincluster
+```
+
+please refer to the [kubectl plugin documentation](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/) for 
+more information.
+
+# Drain Member Cluster connected to a fleet
+
+After following the steps above to build the `draincluster` tool as a kubectl plugin, you can use it to remove all 
+resources propagated to the member cluster from the hub cluster by any `Placement` resource. This is useful when you 
+want to temporarily move all workloads off a member cluster in preparation for an event like upgrade or reconfiguration.
+
+The `draincluster` tool can be used to drain a member cluster by running the following command:
+
+```
+kubectl draincluster --hubClusterContext <hub-cluster-context> --clusterName <memberClusterName>
+```
+
+the tool currently is a go program that takes the hub cluster context and the member cluster name as arguments.
+
+- The `--hubClusterContext` flag specifies the context of the hub cluster
+- The `--clusterName` flag specifies the name of the member cluster to drain.
+
+the user can run the following command to identify the context of the hub cluster:
+
+```
+kubectl config get-contexts
+```
+
+the output of the command will look like this:
+
+```
+CURRENT   NAME               CLUSTER            AUTHINFO                                            NAMESPACE         
+*         hub                hub                clusterUser_clusterResourceGroup_hub   
+```
+
+Here you can see that the context of the hub cluster is called `hub` under the `NAME` column.
+
+The command adds a `Taint` to the `MemberCluster` resource of the member cluster to prevent any new resources from being 
+propagated to the member cluster. Then it creates `Eviction` objects for all the `Placement` objects that have propagated 
+resources to the member cluster.
+
+>> **Note**: The `draincluster` tool is a best-effort mechanism at the moment, so once the command is run successfully
+> the user must verify if all resources propagated by `Placement` resources are removed from the member cluster.
+> Re-running the command is safe and is recommended if the user notices any resources still present on the member cluster.