test(e2e): remove fixed-sleep flakiness across e2e specs (#427)

Author: AlinsRan

test/e2e/crds/v1alpha1/backendtrafficpolicy.go

@@ -20,7 +20,6 @@ package v1alpha1
 import (
 	"context"
 	"fmt"
-	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -63,17 +62,14 @@ spec:
 		By("create GatewayProxy")
 		err = s.CreateResourceFromString(s.GetGatewayProxySpec())
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
-		time.Sleep(5 * time.Second)
 
 		By("create GatewayClass")
 		err = s.CreateResourceFromString(s.GetGatewayClassYaml())
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
-		time.Sleep(5 * time.Second)
 
 		By("create Gateway")
 		err = s.CreateResourceFromString(s.GetGatewayYaml())
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
-		time.Sleep(5 * time.Second)
 
 		By("create HTTPRoute")
 		s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, fmt.Sprintf(defaultHTTPRoute, s.Namespace(), s.Namespace()))
@@ -232,28 +228,28 @@ spec:
 					scaffold.WithExpectedStatus(200),
 				},
 			})
-			time.Sleep(2 * time.Second)
-
-			ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background())
-			Expect(err).ToNot(HaveOccurred(), "listing upstreams")
-			Expect(ups).NotTo(BeEmpty(), "upstreams should not be empty")
-
-			var target *adctypes.Upstream
-			for _, u := range ups {
-				if u.Checks != nil {
-					target = u
-					break
+			Eventually(func(g Gomega) {
+				ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background())
+				g.Expect(err).ToNot(HaveOccurred(), "listing upstreams")
+				g.Expect(ups).NotTo(BeEmpty(), "upstreams should not be empty")
+
+				var target *adctypes.Upstream
+				for _, u := range ups {
+					if u.Checks != nil {
+						target = u
+						break
+					}
 				}
-			}
-			Expect(target).NotTo(BeNil(), "upstream with health check should exist")
-			Expect(target.Checks.Active).NotTo(BeNil(), "active health check should be configured")
-			Expect(target.Checks.Active.HTTPPath).To(Equal("/get"), "active health check http path")
-			Expect(target.Checks.Active.Healthy.Interval).To(Equal(1), "active healthy interval")
-			Expect(target.Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "active healthy http codes")
-			Expect(target.Checks.Active.Unhealthy.Interval).To(Equal(1), "active unhealthy interval")
-			Expect(target.Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "active unhealthy http failures")
-			Expect(target.Checks.Active.Unhealthy.HTTPStatuses).To(Equal([]int{500}), "active unhealthy http codes")
-			Expect(target.Checks.Passive).To(BeNil(), "passive health check should not be configured")
+				g.Expect(target).NotTo(BeNil(), "upstream with health check should exist")
+				g.Expect(target.Checks.Active).NotTo(BeNil(), "active health check should be configured")
+				g.Expect(target.Checks.Active.HTTPPath).To(Equal("/get"), "active health check http path")
+				g.Expect(target.Checks.Active.Healthy.Interval).To(Equal(1), "active healthy interval")
+				g.Expect(target.Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "active healthy http codes")
+				g.Expect(target.Checks.Active.Unhealthy.Interval).To(Equal(1), "active unhealthy interval")
+				g.Expect(target.Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "active unhealthy http failures")
+				g.Expect(target.Checks.Active.Unhealthy.HTTPStatuses).To(Equal([]int{500}), "active unhealthy http codes")
+				g.Expect(target.Checks.Passive).To(BeNil(), "passive health check should not be configured")
+			}).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed())
 		})
 
 		It("should configure active and passive health checks on upstream", func() {
@@ -268,31 +264,31 @@ spec:
 					scaffold.WithExpectedStatus(200),
 				},
 			})
-			time.Sleep(2 * time.Second)
-
-			ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background())
-			Expect(err).ToNot(HaveOccurred(), "listing upstreams")
-			Expect(ups).NotTo(BeEmpty(), "upstreams should not be empty")
-
-			var target *adctypes.Upstream
-			for _, u := range ups {
-				if u.Checks != nil && u.Checks.Passive != nil {
-					target = u
-					break
+			Eventually(func(g Gomega) {
+				ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background())
+				g.Expect(err).ToNot(HaveOccurred(), "listing upstreams")
+				g.Expect(ups).NotTo(BeEmpty(), "upstreams should not be empty")
+
+				var target *adctypes.Upstream
+				for _, u := range ups {
+					if u.Checks != nil && u.Checks.Passive != nil {
+						target = u
+						break
+					}
 				}
-			}
-			Expect(target).NotTo(BeNil(), "upstream with active and passive health check should exist")
-
-			// Verify active health check
-			Expect(target.Checks.Active).NotTo(BeNil(), "active health check should be configured")
-			Expect(target.Checks.Active.HTTPPath).To(Equal("/get"), "active health check http path")
-			Expect(target.Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "active healthy http codes")
-			Expect(target.Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "active unhealthy http failures")
-
-			// Verify passive health check
-			Expect(target.Checks.Passive.Healthy.HTTPStatuses).To(Equal([]int{200}), "passive healthy http codes")
-			Expect(target.Checks.Passive.Unhealthy.HTTPStatuses).To(Equal([]int{502, 503}), "passive unhealthy http codes")
-			Expect(target.Checks.Passive.Unhealthy.HTTPFailures).To(Equal(3), "passive unhealthy http failures")
+				g.Expect(target).NotTo(BeNil(), "upstream with active and passive health check should exist")
+
+				// Verify active health check
+				g.Expect(target.Checks.Active).NotTo(BeNil(), "active health check should be configured")
+				g.Expect(target.Checks.Active.HTTPPath).To(Equal("/get"), "active health check http path")
+				g.Expect(target.Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "active healthy http codes")
+				g.Expect(target.Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "active unhealthy http failures")
+
+				// Verify passive health check
+				g.Expect(target.Checks.Passive.Healthy.HTTPStatuses).To(Equal([]int{200}), "passive healthy http codes")
+				g.Expect(target.Checks.Passive.Unhealthy.HTTPStatuses).To(Equal([]int{502, 503}), "passive unhealthy http codes")
+				g.Expect(target.Checks.Passive.Unhealthy.HTTPFailures).To(Equal(3), "passive unhealthy http failures")
+			}).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed())
 		})
 
 		It("should remove health check when policy is deleted", func() {
@@ -307,32 +303,33 @@ spec:
 					scaffold.WithExpectedStatus(200),
 				},
 			})
-			time.Sleep(2 * time.Second)
-
 			// Verify health check is present on the target upstream
-			ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background())
-			Expect(err).ToNot(HaveOccurred())
-			hasHealthCheck := false
-			for _, u := range ups {
-				if u.Checks != nil {
-					hasHealthCheck = true
-					break
+			Eventually(func(g Gomega) {
+				ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background())
+				g.Expect(err).ToNot(HaveOccurred())
+				hasHealthCheck := false
+				for _, u := range ups {
+					if u.Checks != nil {
+						hasHealthCheck = true
+						break
+					}
 				}
-			}
-			Expect(hasHealthCheck).To(BeTrue(), "upstream should have health check before policy deletion")
+				g.Expect(hasHealthCheck).To(BeTrue(), "upstream should have health check before policy deletion")
+			}).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed())
 
 			// Delete the policy
 			err = s.DeleteResourceFromString(policyWithActiveHealthCheck)
 			Expect(err).NotTo(HaveOccurred(), "deleting BackendTrafficPolicy")
-			time.Sleep(3 * time.Second)
 
 			// Verify health check is removed from the target upstream
-			ups, err = s.DefaultDataplaneResource().Upstream().List(context.Background())
-			Expect(err).ToNot(HaveOccurred())
-			Expect(ups).NotTo(BeEmpty(), "upstreams should still exist after policy deletion")
-			for _, u := range ups {
-				Expect(u.Checks).To(BeNil(), "upstream should not have health check after policy deletion")
-			}
+			Eventually(func(g Gomega) {
+				ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background())
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(ups).NotTo(BeEmpty(), "upstreams should still exist after policy deletion")
+				for _, u := range ups {
+					g.Expect(u.Checks).To(BeNil(), "upstream should not have health check after policy deletion")
+				}
+			}).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed())
 		})
 	})
 })
@@ -379,7 +376,6 @@ spec:
 		By("create GatewayProxy")
 		err := s.CreateResourceFromString(s.GetGatewayProxySpec())
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
-		time.Sleep(5 * time.Second)
 
 		By("create IngressClass with GatewayProxy reference")
 		err = s.CreateResourceFromString(fmt.Sprintf(defaultIngressClass, s.GetControllerName(), s.Namespace()))

test/e2e/crds/v1alpha1/consumer.go

@@ -80,17 +80,14 @@ spec:
 		By("create GatewayProxy")
 		err = s.CreateResourceFromString(s.GetGatewayProxySpec())
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
-		time.Sleep(time.Second)
 
 		By("create GatewayClass")
 		err = s.CreateResourceFromString(s.GetGatewayClassYaml())
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
-		time.Sleep(time.Second)
 
 		By("create Gateway")
 		err = s.CreateResourceFromString(s.GetGatewayYaml())
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
-		time.Sleep(time.Second)
 
 		By("create HTTPRoute")
 		s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, fmt.Sprintf(defaultHTTPRoute, s.Namespace()))

test/e2e/crds/v1alpha1/gatewayproxy.go

@@ -117,17 +117,14 @@ spec:
 			err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxySpec, framework.ProviderType, s.AdminKey()))
 		}
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
-		time.Sleep(5 * time.Second)
 
 		By("create GatewayClass")
 		err = s.CreateResourceFromString(s.GetGatewayClassYaml())
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
-		time.Sleep(5 * time.Second)
 
 		By("create Gateway")
 		err = s.CreateResourceFromString(s.GetGatewayYaml())
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
-		time.Sleep(5 * time.Second)
 
 		By("create HTTPRoute")
 		s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, fmt.Sprintf(httpRouteSpec, gatewayName))

test/e2e/crds/v2/basic.go

@@ -67,12 +67,10 @@ var _ = Describe("APISIX Standalone Basic Tests", Label("apisix.apache.org", "v2
 			By("create GatewayProxy")
 			err := s.CreateResourceFromString(s.GetGatewayProxySpec())
 			Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
-			time.Sleep(5 * time.Second)
 
 			By("create IngressClass")
 			err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "")
 			Expect(err).NotTo(HaveOccurred(), "creating IngressClass")
-			time.Sleep(5 * time.Second)
 
 			const apisixRouteSpec = `
 apiVersion: apisix.apache.org/v2

test/e2e/crds/v2/consumer.go

@@ -72,12 +72,10 @@ var _ = Describe("Test ApisixConsumer", Label("apisix.apache.org", "v2", "apisix
 		By("create GatewayProxy")
 		err := s.CreateResourceFromString(s.GetGatewayProxySpec())
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
-		time.Sleep(5 * time.Second)
 
 		By("create IngressClass")
 		err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "")
 		Expect(err).NotTo(HaveOccurred(), "creating IngressClass")
-		time.Sleep(5 * time.Second)
 	})
 
 	Context("Test KeyAuth", func() {