Skip to content

Commit d30e23d

Browse files
camilamacedo86camilamacedo86
authored
(e2e): Enhance e2e test to check workload resilience when catalogs goes away (#3747)
1 parent 6fc19e0 commit d30e23d

1 file changed

Lines changed: 99 additions & 146 deletions

File tree

‎test/e2e/catalog_e2e_test.go‎

Lines changed: 99 additions & 146 deletions
Original file line numberDiff line numberDiff line change
@@ -1864,43 +1864,39 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
18641864

18651865
By("Wait for operator deployment to be ready")
18661866
var operatorDeployment *appsv1.Deployment
1867-
Eventually(func() error {
1867+
Eventually(func(g Gomega) {
1868+
var err error
18681869
operatorDeployment, err = c.GetDeployment(generatedNamespace.GetName(), deploymentName)
1869-
if err != nil {
1870-
return err
1871-
}
1872-
if operatorDeployment.Spec.Replicas == nil || *operatorDeployment.Spec.Replicas == 0 {
1873-
return fmt.Errorf("deployment replicas is not set")
1874-
}
1875-
if operatorDeployment.Status.AvailableReplicas != *operatorDeployment.Spec.Replicas {
1876-
return fmt.Errorf("deployment %s not ready: %d/%d replicas available",
1877-
deploymentName,
1878-
operatorDeployment.Status.AvailableReplicas,
1879-
*operatorDeployment.Spec.Replicas)
1880-
}
1881-
if operatorDeployment.Status.ReadyReplicas != *operatorDeployment.Spec.Replicas {
1882-
return fmt.Errorf("deployment %s not ready: %d/%d replicas ready",
1883-
deploymentName,
1884-
operatorDeployment.Status.ReadyReplicas,
1885-
*operatorDeployment.Spec.Replicas)
1886-
}
1887-
return nil
1888-
}, pollDuration, pollInterval).Should(Succeed())
1870+
g.Expect(err).ShouldNot(HaveOccurred())
1871+
g.Expect(operatorDeployment.Spec.Replicas).NotTo(BeNil())
1872+
g.Expect(*operatorDeployment.Spec.Replicas).NotTo(BeZero())
1873+
g.Expect(operatorDeployment.Status.AvailableReplicas).To(Equal(*operatorDeployment.Spec.Replicas),
1874+
"deployment %s not ready: %d/%d replicas available",
1875+
deploymentName,
1876+
operatorDeployment.Status.AvailableReplicas,
1877+
*operatorDeployment.Spec.Replicas)
1878+
g.Expect(operatorDeployment.Status.ReadyReplicas).To(Equal(*operatorDeployment.Spec.Replicas),
1879+
"deployment %s not ready: %d/%d replicas ready",
1880+
deploymentName,
1881+
operatorDeployment.Status.ReadyReplicas,
1882+
*operatorDeployment.Spec.Replicas)
1883+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
18891884

18901885
By("Record deployment state before catalog deletion")
18911886
deploymentUID := operatorDeployment.UID
18921887
expectedReplicas := *operatorDeployment.Spec.Replicas
18931888

18941889
By("Verify ServiceAccount, Role, and RoleBinding created by OLM")
18951890
var serviceAccount *corev1.ServiceAccount
1896-
Eventually(func() error {
1891+
Eventually(func(g Gomega) {
1892+
var err error
18971893
serviceAccount, err = c.KubernetesInterface().CoreV1().ServiceAccounts(generatedNamespace.GetName()).Get(
18981894
context.Background(),
18991895
serviceAccountName,
19001896
metav1.GetOptions{},
19011897
)
1902-
return err
1903-
}, pollDuration, pollInterval).Should(Succeed())
1898+
g.Expect(err).ShouldNot(HaveOccurred())
1899+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
19041900
serviceAccountUID := serviceAccount.UID
19051901

19061902
// Roles and RoleBindings are owned by the CSV with generated names, so we list them by owner
@@ -1911,144 +1907,99 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
19111907
})
19121908

19131909
var roleList *rbacv1.RoleList
1914-
Eventually(func() error {
1910+
Eventually(func(g Gomega) {
1911+
var err error
19151912
roleList, err = c.KubernetesInterface().RbacV1().Roles(generatedNamespace.GetName()).List(
19161913
context.Background(),
19171914
metav1.ListOptions{LabelSelector: ownerSelector.String()},
19181915
)
1919-
if err != nil {
1920-
return err
1921-
}
1922-
if len(roleList.Items) == 0 {
1923-
return fmt.Errorf("no roles found owned by CSV")
1924-
}
1925-
return nil
1926-
}, pollDuration, pollInterval).Should(Succeed())
1916+
g.Expect(err).ShouldNot(HaveOccurred())
1917+
g.Expect(roleList.Items).ToNot(BeEmpty(), "no roles found owned by CSV")
1918+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
19271919
roleUID := roleList.Items[0].UID
19281920

19291921
var roleBindingList *rbacv1.RoleBindingList
1930-
Eventually(func() error {
1922+
Eventually(func(g Gomega) {
1923+
var err error
19311924
roleBindingList, err = c.KubernetesInterface().RbacV1().RoleBindings(generatedNamespace.GetName()).List(
19321925
context.Background(),
19331926
metav1.ListOptions{LabelSelector: ownerSelector.String()},
19341927
)
1935-
if err != nil {
1936-
return err
1937-
}
1938-
if len(roleBindingList.Items) == 0 {
1939-
return fmt.Errorf("no rolebindings found owned by CSV")
1940-
}
1941-
return nil
1942-
}, pollDuration, pollInterval).Should(Succeed())
1928+
g.Expect(err).ShouldNot(HaveOccurred())
1929+
g.Expect(roleBindingList.Items).ToNot(BeEmpty(), "no rolebindings found owned by CSV")
1930+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
19431931
roleBindingUID := roleBindingList.Items[0].UID
19441932

19451933
By("Delete catalog source")
19461934
err = crc.OperatorsV1alpha1().CatalogSources(catalogSource.GetNamespace()).Delete(context.Background(), catalogSource.GetName(), metav1.DeleteOptions{})
19471935
Expect(err).ShouldNot(HaveOccurred())
19481936

19491937
By("Wait for catalog source to be removed")
1950-
Eventually(func() error {
1938+
Eventually(func(g Gomega) {
19511939
_, err := crc.OperatorsV1alpha1().CatalogSources(catalogSource.GetNamespace()).Get(context.Background(), catalogSource.GetName(), metav1.GetOptions{})
1952-
if err == nil {
1953-
return fmt.Errorf("catalog source still exists")
1954-
}
1955-
if !k8serror.IsNotFound(err) {
1956-
return err
1957-
}
1958-
return nil
1959-
}, pollDuration, pollInterval).Should(Succeed())
1940+
g.Expect(k8serror.IsNotFound(err)).To(BeTrue(), "catalog source should be deleted")
1941+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
19601942

19611943
By("Wait for catalog source pod to be deleted")
1962-
Eventually(func() error {
1944+
Eventually(func(g Gomega) {
19631945
listOpts := metav1.ListOptions{
19641946
LabelSelector: "olm.catalogSource=" + catalogSourceName,
19651947
}
19661948
pods, err := c.KubernetesInterface().CoreV1().Pods(catalogSource.GetNamespace()).List(context.Background(), listOpts)
1967-
if err != nil {
1968-
return err
1969-
}
1970-
if len(pods.Items) > 0 {
1971-
return fmt.Errorf("catalog source pod still exists: %d pods found", len(pods.Items))
1972-
}
1973-
return nil
1974-
}, pollDuration, pollInterval).Should(Succeed())
1949+
g.Expect(err).ShouldNot(HaveOccurred())
1950+
g.Expect(pods.Items).To(BeEmpty(), "catalog source pod should be deleted")
1951+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
19751952

19761953
By("Verify subscription behavior after catalog deletion")
1977-
Eventually(func() error {
1954+
Eventually(func(g Gomega) {
19781955
sub, err := crc.OperatorsV1alpha1().Subscriptions(generatedNamespace.GetName()).Get(
19791956
context.Background(),
19801957
subscriptionName,
19811958
metav1.GetOptions{},
19821959
)
1983-
if err != nil {
1984-
return fmt.Errorf("failed to get subscription: %w", err)
1985-
}
1960+
g.Expect(err).ShouldNot(HaveOccurred(), "failed to get subscription")
19861961

19871962
// Subscription should still track the installed CSV
1988-
if sub.Status.InstalledCSV != packageStable {
1989-
return fmt.Errorf("subscription InstalledCSV changed from %s to %s", packageStable, sub.Status.InstalledCSV)
1990-
}
1963+
g.Expect(sub.Status.InstalledCSV).To(Equal(packageStable), "subscription InstalledCSV should not change")
19911964

19921965
// Verify catalog health behavior: if the deleted catalog is still in the health list,
19931966
// it should be marked as unhealthy. If it's been removed from the list, that's also acceptable.
19941967
for _, health := range sub.Status.CatalogHealth {
19951968
if health.CatalogSourceRef != nil && health.CatalogSourceRef.Name == catalogSourceName {
1996-
if health.Healthy {
1997-
return fmt.Errorf("subscription still reports deleted catalog %s as healthy", catalogSourceName)
1998-
}
1969+
g.Expect(health.Healthy).To(BeFalse(), "subscription should not report deleted catalog %s as healthy", catalogSourceName)
19991970
}
20001971
}
2001-
2002-
return nil
2003-
}, pollDuration, pollInterval).Should(Succeed())
1972+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
20041973

20051974
By("Verify CSV remains in succeeded state after catalog deletion")
2006-
Consistently(func() error {
1975+
Consistently(func(g Gomega) {
20071976
fetchedCSV, err := crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Get(
20081977
context.Background(),
20091978
installedCSV.GetName(),
20101979
metav1.GetOptions{},
20111980
)
2012-
if err != nil {
2013-
return fmt.Errorf("failed to get CSV: %w", err)
2014-
}
2015-
if fetchedCSV.Status.Phase != v1alpha1.CSVPhaseSucceeded {
2016-
return fmt.Errorf("CSV phase is %s, expected Succeeded", fetchedCSV.Status.Phase)
2017-
}
2018-
return nil
2019-
}, 3*time.Minute, pollInterval).Should(Succeed())
1981+
g.Expect(err).ShouldNot(HaveOccurred(), "failed to get CSV")
1982+
g.Expect(fetchedCSV.Status.Phase).To(Equal(v1alpha1.CSVPhaseSucceeded), "CSV should remain in Succeeded state")
1983+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
20201984

20211985
By("Verify deployment remains healthy and unchanged")
2022-
Consistently(func() error {
1986+
Consistently(func(g Gomega) {
20231987
deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
2024-
if err != nil {
2025-
return fmt.Errorf("failed to get deployment: %w", err)
2026-
}
2027-
if deployment.UID != deploymentUID {
2028-
return fmt.Errorf("deployment was recreated")
2029-
}
2030-
if deployment.Spec.Replicas == nil {
2031-
return fmt.Errorf("deployment replicas is nil")
2032-
}
2033-
if deployment.Status.AvailableReplicas != expectedReplicas {
2034-
return fmt.Errorf("available replicas: got %d, want %d", deployment.Status.AvailableReplicas, expectedReplicas)
2035-
}
2036-
if deployment.Status.ReadyReplicas != expectedReplicas {
2037-
return fmt.Errorf("ready replicas: got %d, want %d", deployment.Status.ReadyReplicas, expectedReplicas)
2038-
}
2039-
return nil
2040-
}, 3*time.Minute, pollInterval).Should(Succeed())
1988+
g.Expect(err).ShouldNot(HaveOccurred(), "failed to get deployment")
1989+
g.Expect(deployment.UID).To(Equal(deploymentUID), "deployment should not be recreated")
1990+
g.Expect(deployment.Spec.Replicas).NotTo(BeNil(), "deployment replicas should be set")
1991+
g.Expect(deployment.Status.AvailableReplicas).To(Equal(expectedReplicas), "available replicas should match expected")
1992+
g.Expect(deployment.Status.ReadyReplicas).To(Equal(expectedReplicas), "ready replicas should match expected")
1993+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
20411994

20421995
By("Test OLM config management - add environment variable via subscription")
2043-
Eventually(func() error {
1996+
Eventually(func(g Gomega) {
20441997
sub, err := crc.OperatorsV1alpha1().Subscriptions(generatedNamespace.GetName()).Get(
20451998
context.Background(),
20461999
subscriptionName,
20472000
metav1.GetOptions{},
20482001
)
2049-
if err != nil {
2050-
return err
2051-
}
2002+
g.Expect(err).ShouldNot(HaveOccurred())
20522003

20532004
if sub.Spec.Config == nil {
20542005
sub.Spec.Config = &v1alpha1.SubscriptionConfig{}
@@ -2062,26 +2013,25 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
20622013
sub,
20632014
metav1.UpdateOptions{},
20642015
)
2065-
return err
2066-
}, pollDuration, pollInterval).Should(Succeed())
2016+
g.Expect(err).ShouldNot(HaveOccurred())
2017+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
20672018

20682019
By("Wait for deployment to have the environment variable")
2069-
Eventually(func() error {
2020+
Eventually(func(g Gomega) {
20702021
deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
2071-
if err != nil {
2072-
return err
2073-
}
2074-
if len(deployment.Spec.Template.Spec.Containers) == 0 {
2075-
return fmt.Errorf("no containers in deployment")
2076-
}
2022+
g.Expect(err).ShouldNot(HaveOccurred())
2023+
g.Expect(deployment.Spec.Template.Spec.Containers).NotTo(BeEmpty(), "deployment should have containers")
2024+
20772025
container := deployment.Spec.Template.Spec.Containers[0]
2026+
envVarFound := false
20782027
for _, env := range container.Env {
20792028
if env.Name == "TEST_ENV_VAR" && env.Value == "test-value" {
2080-
return nil
2029+
envVarFound = true
2030+
break
20812031
}
20822032
}
2083-
return fmt.Errorf("TEST_ENV_VAR not found in deployment")
2084-
}, pollDuration, pollInterval).Should(Succeed())
2033+
g.Expect(envVarFound).To(BeTrue(), "TEST_ENV_VAR should be found in deployment")
2034+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
20852035

20862036
By("Delete the operator deployment to test OLM reconciliation")
20872037
err = c.KubernetesInterface().AppsV1().Deployments(generatedNamespace.GetName()).Delete(
@@ -2092,43 +2042,44 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
20922042
Expect(err).ShouldNot(HaveOccurred())
20932043

20942044
By("Wait for deployment to be deleted")
2095-
Eventually(func() error {
2045+
Eventually(func(g Gomega) {
20962046
_, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
2097-
if err == nil {
2098-
return fmt.Errorf("deployment still exists")
2099-
}
2100-
if !k8serror.IsNotFound(err) {
2101-
return err
2102-
}
2103-
return nil
2104-
}, pollDuration, pollInterval).Should(Succeed())
2047+
g.Expect(k8serror.IsNotFound(err)).To(BeTrue(), "deployment should be deleted")
2048+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
21052049

21062050
By("Wait for OLM to recreate the deployment")
2107-
Eventually(func() error {
2051+
// Use a longer timeout here since OLM needs to:
2052+
// 1. Detect the deployment deletion (via watch or reconciliation loop)
2053+
// 2. Recreate the deployment
2054+
// 3. Wait for the deployment to become ready (pull image, start pod, etc.)
2055+
// In slow/busy CI environments, this can take longer than the standard 5 minutes
2056+
Eventually(func(g Gomega) {
21082057
deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
2109-
if err != nil {
2110-
return fmt.Errorf("deployment not recreated yet: %w", err)
2111-
}
2112-
if deployment.UID == deploymentUID {
2113-
return fmt.Errorf("deployment UID unchanged, not recreated")
2114-
}
2115-
if deployment.Spec.Replicas == nil {
2116-
return fmt.Errorf("deployment replicas is nil")
2117-
}
2118-
if deployment.Status.AvailableReplicas != expectedReplicas {
2119-
return fmt.Errorf("available replicas: got %d, want %d", deployment.Status.AvailableReplicas, expectedReplicas)
2120-
}
2121-
if deployment.Status.ReadyReplicas != expectedReplicas {
2122-
return fmt.Errorf("ready replicas: got %d, want %d", deployment.Status.ReadyReplicas, expectedReplicas)
2123-
}
2124-
return nil
2125-
}, pollDuration, pollInterval).Should(Succeed())
2058+
g.Expect(err).ShouldNot(HaveOccurred(), "deployment should exist")
2059+
g.Expect(deployment.UID).NotTo(Equal(deploymentUID), "deployment should have new UID (recreated)")
2060+
g.Expect(deployment.Spec.Replicas).NotTo(BeNil(), "deployment replicas should be set")
2061+
g.Expect(*deployment.Spec.Replicas).NotTo(BeZero(), "deployment replicas should not be zero")
2062+
2063+
// Check that pods are actually ready, not just that the deployment exists
2064+
g.Expect(deployment.Status.AvailableReplicas).To(Equal(expectedReplicas),
2065+
"deployment should have %d available replicas, got %d", expectedReplicas, deployment.Status.AvailableReplicas)
2066+
g.Expect(deployment.Status.ReadyReplicas).To(Equal(expectedReplicas),
2067+
"deployment should have %d ready replicas, got %d", expectedReplicas, deployment.Status.ReadyReplicas)
2068+
g.Expect(deployment.Status.UpdatedReplicas).To(Equal(expectedReplicas),
2069+
"deployment should have %d updated replicas, got %d", expectedReplicas, deployment.Status.UpdatedReplicas)
2070+
}).WithTimeout(8 * time.Minute).WithPolling(5 * time.Second).Should(Succeed())
21262071

21272072
By("Verify all resources were recreated by OLM with correct configuration")
2128-
recreatedDeployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
2129-
Expect(err).ShouldNot(HaveOccurred())
2130-
Expect(recreatedDeployment.UID).ToNot(Equal(deploymentUID), "deployment should have been recreated with new UID")
2073+
// Re-fetch the deployment to get the latest state after recreation
2074+
var recreatedDeployment *appsv1.Deployment
2075+
Eventually(func(g Gomega) {
2076+
var err error
2077+
recreatedDeployment, err = c.GetDeployment(generatedNamespace.GetName(), deploymentName)
2078+
g.Expect(err).ShouldNot(HaveOccurred())
2079+
g.Expect(recreatedDeployment.UID).ToNot(Equal(deploymentUID), "deployment should have been recreated with new UID")
2080+
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
21312081

2082+
// Verify ServiceAccount was NOT recreated (should have same UID)
21322083
recreatedServiceAccount, err := c.KubernetesInterface().CoreV1().ServiceAccounts(generatedNamespace.GetName()).Get(
21332084
context.Background(),
21342085
serviceAccountName,
@@ -2137,6 +2088,7 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
21372088
Expect(err).ShouldNot(HaveOccurred())
21382089
Expect(recreatedServiceAccount.UID).To(Equal(serviceAccountUID), "serviceaccount should not have been recreated (same UID)")
21392090

2091+
// Verify Role was NOT recreated (should have same UID)
21402092
recreatedRoleList, err := c.KubernetesInterface().RbacV1().Roles(generatedNamespace.GetName()).List(
21412093
context.Background(),
21422094
metav1.ListOptions{LabelSelector: ownerSelector.String()},
@@ -2145,6 +2097,7 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
21452097
Expect(len(recreatedRoleList.Items)).To(BeNumerically(">", 0), "at least one role should exist")
21462098
Expect(recreatedRoleList.Items[0].UID).To(Equal(roleUID), "role should not have been recreated (same UID)")
21472099

2100+
// Verify RoleBinding was NOT recreated (should have same UID)
21482101
recreatedRoleBindingList, err := c.KubernetesInterface().RbacV1().RoleBindings(generatedNamespace.GetName()).List(
21492102
context.Background(),
21502103
metav1.ListOptions{LabelSelector: ownerSelector.String()},

0 commit comments

Comments
 (0)