fix(store): automatically remove offlined store (#6497)

Author: liubog2008

pkg/controllers/common/cond.go

@@ -17,6 +17,9 @@ package common
 import (
 	"slices"
 
+	"k8s.io/apimachinery/pkg/api/meta"
+
+	"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
 	coreutil "github.com/pingcap/tidb-operator/pkg/apiutil/core/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/client"
 	"github.com/pingcap/tidb-operator/pkg/runtime"
@@ -89,3 +92,17 @@ func CondFeatureGatesIsNotSynced[
 		return !slices.Equal(coreutil.Features[S](obj), coreutil.EnabledFeatures(cluster))
 	})
 }
+
+func CondObjectIsNotDeletingButOfflined[
+	S scope.Instance[F, T],
+	F client.Object,
+	T runtime.Instance,
+](state ObjectState[F]) task.Condition {
+	return task.CondFunc(func() bool {
+		if !state.Object().GetDeletionTimestamp().IsZero() {
+			return false
+		}
+		conds := coreutil.StatusConditions[S](state.Object())
+		return meta.IsStatusConditionTrue(conds, v1alpha1.StoreOfflinedConditionType)
+	})
+}

pkg/controllers/common/cond_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
 	metav1alpha1 "github.com/pingcap/tidb-operator/api/v2/meta/v1alpha1"
@@ -300,3 +301,49 @@ func TestCondFeatureGatesIsNotSynced(t *testing.T) {
 		})
 	}
 }
+
+func TestCondObjectIsNotDeletingButOfflined(t *testing.T) {
+	cases := []struct {
+		desc         string
+		state        ObjectState[*v1alpha1.TiKV]
+		expectedCond bool
+	}{
+		{
+			desc: "cond is false, not offlined",
+			state: newFakeObjectState(
+				fake.FakeObj[v1alpha1.TiKV]("test"),
+			),
+		},
+		{
+			desc: "cond is false, deleting",
+			state: newFakeObjectState(
+				fake.FakeObj("test", fake.DeleteNow[v1alpha1.TiKV]()),
+			),
+		},
+		{
+			desc: "cond is true",
+			state: newFakeObjectState(
+				fake.FakeObj("test", func(obj *v1alpha1.TiKV) *v1alpha1.TiKV {
+					obj.Status.Conditions = []metav1.Condition{
+						{
+							Type:   v1alpha1.StoreOfflinedConditionType,
+							Status: metav1.ConditionTrue,
+						},
+					}
+					return obj
+				}),
+			),
+			expectedCond: true,
+		},
+	}
+
+	for i := range cases {
+		c := &cases[i]
+		t.Run(c.desc, func(tt *testing.T) {
+			tt.Parallel()
+
+			cond := CondObjectIsNotDeletingButOfflined[scope.TiKV](c.state)
+			assert.Equal(tt, c.expectedCond, cond.Satisfy(), c.desc)
+		})
+	}
+}

pkg/controllers/common/task.go

@@ -21,6 +21,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
@@ -318,3 +319,25 @@ func TaskTrack[
 		return task.Complete().With("track instance %s with group %v", state.Key(), group)
 	})
 }
+
+func TaskDeleteOfflinedStore[
+	S scope.Instance[F, T],
+	F client.Object,
+	T runtime.Instance,
+](state ObjectState[F], c client.Client) task.Task {
+	return task.NameTaskFunc("DeleteOfflinedStore", func(ctx context.Context) task.Result {
+		obj := state.Object()
+		if !obj.GetDeletionTimestamp().IsZero() {
+			return task.Complete().With("store is deleting")
+		}
+		conds := coreutil.StatusConditions[S](state.Object())
+		if !meta.IsStatusConditionTrue(conds, v1alpha1.StoreOfflinedConditionType) {
+			return task.Complete().With("store is not offlined")
+		}
+		if err := c.Delete(ctx, obj); err != nil {
+			return task.Fail().With("cannot delete store: %v", err)
+		}
+
+		return task.Wait().With("wait until store deletion is watched")
+	})
+}

pkg/controllers/tikv/builder.go

@@ -43,6 +43,11 @@ func (r *Reconciler) NewRunner(state *tasks.ReconcileContext, reporter task.Task
 			tasks.TaskFinalizerDel(state, r.Client),
 		),
 
+		// if instance is deleting and store is removed
+		task.IfBreak(common.CondObjectIsNotDeletingButOfflined[scope.TiKV](state),
+			common.TaskDeleteOfflinedStore[scope.TiKV](state, r.Client),
+		),
+
 		// get pod and check whether the cluster is suspending
 		common.TaskContextPod[scope.TiKV](state, r.Client),
 		// get info from pd