chore: fix the random failure in the FlatInstanceOrdinal test (#9662)

Author: leon-ape

controllers/workloads/instanceset_controller_test.go

@@ -641,50 +641,78 @@ var _ = Describe("InstanceSet Controller", func() {
 	})
 
 	Context("pod naming rule", func() {
-		checkPodOrdinal := func(ordinals []int, exist bool) {
+		checkPodOrdinal := func(ordinals []int, checkFunc func(podKey types.NamespacedName)) {
 			for _, ordinal := range ordinals {
 				podKey := types.NamespacedName{
 					Namespace: itsObj.Namespace,
 					Name:      fmt.Sprintf("%v-%v", itsObj.Name, ordinal),
 				}
-				if exist {
-					Eventually(testapps.CheckObjExists(&testCtx, podKey, &corev1.Pod{}, true)).Should(Succeed())
-				} else {
-					Eventually(testapps.CheckObjExists(&testCtx, podKey, &corev1.Pod{}, false)).Should(Succeed())
-				}
+				checkFunc(podKey)
 			}
 		}
 
+		eventuallyExist := func(podKey types.NamespacedName) {
+			Eventually(testapps.CheckObjExists(&testCtx, podKey, &corev1.Pod{}, true)).Should(Succeed())
+		}
+
+		eventuallyNotExist := func(podKey types.NamespacedName) {
+			Eventually(testapps.CheckObjExists(&testCtx, podKey, &corev1.Pod{}, false)).Should(Succeed())
+		}
+
+		consistentlyExist := func(podKey types.NamespacedName) {
+			Consistently(testapps.CheckObjExists(&testCtx, podKey, &corev1.Pod{}, true)).Should(Succeed())
+		}
+
+		consistentlyNotExist := func(podKey types.NamespacedName) {
+			Consistently(testapps.CheckObjExists(&testCtx, podKey, &corev1.Pod{}, false)).Should(Succeed())
+		}
+
 		It("works with FlatInstanceOrdinal", func() {
 			createITSObj(itsName, func(f *testapps.MockInstanceSetFactory) {
 				f.SetFlatInstanceOrdinal(true)
 				f.SetPodManagementPolicy(appsv1.ParallelPodManagement)
 				f.SetReplicas(3)
 			})
 
-			checkPodOrdinal([]int{0, 1, 2}, true)
+			checkPodOrdinal([]int{0, 1, 2}, eventuallyExist)
 			mockPodReady(itsObj.Name+"-0", itsObj.Name+"-1", itsObj.Name+"-2")
+			By("check its status")
+			Eventually(testapps.CheckObj(&testCtx, itsKey, func(g Gomega, its *workloads.InstanceSet) {
+				g.Expect(its.Status.Ordinals).Should(HaveExactElements(int32(0), int32(1), int32(2)))
+			})).Should(Succeed())
 
 			// offline one instance
-			Eventually(testapps.GetAndChangeObj(&testCtx, itsKey, func(its *workloads.InstanceSet) {
+			Expect(testapps.GetAndChangeObj(&testCtx, itsKey, func(its *workloads.InstanceSet) {
 				its.Spec.Replicas = ptr.To[int32](2)
 				its.Spec.OfflineInstances = []string{itsObj.Name + "-1"}
+			})()).Should(Succeed())
+			checkPodOrdinal([]int{1}, eventuallyNotExist)
+			By("check its status")
+			Eventually(testapps.CheckObj(&testCtx, itsKey, func(g Gomega, its *workloads.InstanceSet) {
+				g.Expect(its.Status.Ordinals).Should(HaveExactElements(int32(0), int32(2)))
 			})).Should(Succeed())
-			checkPodOrdinal([]int{1}, false)
 
 			// scale up
-			Eventually(testapps.GetAndChangeObj(&testCtx, itsKey, func(its *workloads.InstanceSet) {
+			Expect(testapps.GetAndChangeObj(&testCtx, itsKey, func(its *workloads.InstanceSet) {
 				its.Spec.Replicas = ptr.To[int32](4)
-			})).Should(Succeed())
-			checkPodOrdinal([]int{0, 2, 3, 4}, true)
+			})()).Should(Succeed())
+			checkPodOrdinal([]int{0, 2, 3, 4}, eventuallyExist)
 			mockPodReady(itsObj.Name+"-3", itsObj.Name+"-4")
+			By("check its status")
+			Eventually(testapps.CheckObj(&testCtx, itsKey, func(g Gomega, its *workloads.InstanceSet) {
+				g.Expect(its.Status.Ordinals).Should(HaveExactElements(int32(0), int32(2), int32(3), int32(4)))
+			})).Should(Succeed())
 
-			// delete OfflineInstances will not affect running instance
-			Eventually(testapps.GetAndChangeObj(&testCtx, itsKey, func(its *workloads.InstanceSet) {
+			// delete OfflineInstances will not affect running instances
+			Expect(testapps.GetAndChangeObj(&testCtx, itsKey, func(its *workloads.InstanceSet) {
 				its.Spec.OfflineInstances = []string{}
+			})()).Should(Succeed())
+			checkPodOrdinal([]int{0, 2, 3, 4}, consistentlyExist)
+			checkPodOrdinal([]int{1}, consistentlyNotExist)
+			By("check its status")
+			Consistently(testapps.CheckObj(&testCtx, itsKey, func(g Gomega, its *workloads.InstanceSet) {
+				g.Expect(its.Status.Ordinals).Should(HaveExactElements(int32(0), int32(2), int32(3), int32(4)))
 			})).Should(Succeed())
-			checkPodOrdinal([]int{0, 2, 3, 4}, true)
-			checkPodOrdinal([]int{1}, false)
 		})
 	})
 })

pkg/controller/instanceset/reconciler_status.go

@@ -21,6 +21,7 @@ package instanceset
 
 import (
 	"encoding/json"
+	"slices"
 	"sort"
 	"time"
 
@@ -145,6 +146,7 @@ func (r *statusReconciler) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilder
 	}
 	its.Status.Replicas = replicas
 	its.Status.Ordinals = ordinals
+	slices.Sort(its.Status.Ordinals)
 	its.Status.ReadyReplicas = readyReplicas
 	its.Status.AvailableReplicas = availableReplicas
 	its.Status.CurrentReplicas = currentReplicas
@@ -215,6 +217,7 @@ func buildTemplatesStatus(template2TemplatesStatus map[string]*workloads.Instanc
 		if len(templateName) == 0 {
 			continue
 		}
+		slices.Sort(templateStatus.Ordinals)
 		templatesStatus = append(templatesStatus, *templateStatus)
 	}
 	sort.Slice(templatesStatus, func(i, j int) bool {

pkg/controller/instanceset2/reconciler_status.go

@@ -21,6 +21,7 @@ package instanceset2
 
 import (
 	"encoding/json"
+	"slices"
 	"sort"
 	"time"
 
@@ -134,6 +135,7 @@ func (r *statusReconciler) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilder
 	}
 	its.Status.Replicas = replicas
 	its.Status.Ordinals = ordinals
+	slices.Sort(its.Status.Ordinals)
 	its.Status.ReadyReplicas = readyReplicas
 	its.Status.AvailableReplicas = availableReplicas
 	its.Status.CurrentReplicas = currentReplicas
@@ -204,6 +206,7 @@ func buildTemplatesStatus(template2TemplatesStatus map[string]*workloads.Instanc
 		if len(templateName) == 0 {
 			continue
 		}
+		slices.Sort(templateStatus.Ordinals)
 		templatesStatus = append(templatesStatus, *templateStatus)
 	}
 	sort.Slice(templatesStatus, func(i, j int) bool {