From 7ef3d28fdb4e9785b544b387e6bc8f20590c7f36 Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Wed, 4 Jun 2025 07:24:26 +0200 Subject: [PATCH] (chore) improve fv test --- test/fv/auto_deploy_dependencies_test.go | 4 +- test/fv/autoscaler_test.go | 81 +++++---- test/fv/continue_on_error_test.go | 103 ++++++------ test/fv/dependencies_test.go | 73 ++++---- test/fv/drift_test.go | 166 ++++++++++++------- test/fv/dryrun_test.go | 159 ++++++++++-------- test/fv/helm_options_test.go | 128 ++++++++------ test/fv/helm_test.go | 77 +++++---- test/fv/helm_with_corev1list_test.go | 4 +- test/fv/kustomize_test.go | 30 ++-- test/fv/kustomize_with_configmap_test.go | 40 +++-- test/fv/leave_helm_charts_test.go | 35 ++-- test/fv/leave_policies_test.go | 27 ++- test/fv/lua_instantiation_test.go | 45 +++-- test/fv/missing_reference_test.go | 27 ++- test/fv/oci_test.go | 35 ++-- test/fv/optional_referenced_resource_test.go | 76 +++++---- test/fv/patch_multiple_resources_test.go | 43 +++-- test/fv/paused_test.go | 27 ++- test/fv/profile_local_test.go | 24 +-- test/fv/reloader_test.go | 24 ++- test/fv/resource_per_cluster_test.go | 37 +++-- test/fv/resource_test.go | 37 +++-- test/fv/stale_resource_test.go | 27 ++- test/fv/tenant-admin_test.go | 39 +++-- test/fv/tier_test.go | 75 +++++---- 26 files changed, 861 insertions(+), 582 deletions(-) diff --git a/test/fv/auto_deploy_dependencies_test.go b/test/fv/auto_deploy_dependencies_test.go index 9b33a1ce..83051e2e 100644 --- a/test/fv/auto_deploy_dependencies_test.go +++ b/test/fv/auto_deploy_dependencies_test.go @@ -47,7 +47,7 @@ metadata: The ':latest' tag is mutable and can lead to unexpected errors if the image changes. A best practice is to use an immutable tag that maps to a specific version of an application Pod. This policy validates that the image - specifies a tag and that it is not called latest. + specifies a tag and that it is not called latest. spec: validationFailureAction: Audit background: true @@ -98,7 +98,7 @@ var _ = Describe("Feature", func() { It("With AutoDeployDependencies set Sveltos resolves all prerequesities", Label("NEW-FV"), func() { Byf("Create a ClusterProfile matching NO Cluster") // clusterSelector is not set helmClusterProfile := getClusterProfile(namePrefix, map[string]string{}) - Byf("Update ClusterProfile %s to deploy Kyverno helm chart", helmClusterProfile.Name) + Byf("Create ClusterProfile %s to deploy Kyverno helm chart", helmClusterProfile.Name) helmClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ { RepositoryURL: "https://kyverno.github.io/kyverno/", diff --git a/test/fv/autoscaler_test.go b/test/fv/autoscaler_test.go index eb0aa186..63a35807 100644 --- a/test/fv/autoscaler_test.go +++ b/test/fv/autoscaler_test.go @@ -26,6 +26,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -72,12 +73,17 @@ var _ = Describe("Feature", func() { It("Deploy resources in the management cluster and the managed cluster", Label("FV", "EXTENDED"), func() { Byf("Extend sveltos addon-controller permission in the management cluster") - clusterRole := &rbacv1.ClusterRole{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "addon-controller-role-extra"}, clusterRole)).To(Succeed()) - clusterRole.Rules = []rbacv1.PolicyRule{ - {Verbs: []string{"*"}, APIGroups: []string{""}, Resources: []string{"serviceaccounts", "secrets"}}, - } - Expect(k8sClient.Update(context.TODO(), clusterRole)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + clusterRole := &rbacv1.ClusterRole{} + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: "addon-controller-role-extra"}, clusterRole)).To(Succeed()) + clusterRole.Rules = []rbacv1.PolicyRule{ + {Verbs: []string{"*"}, APIGroups: []string{""}, Resources: []string{"serviceaccounts", "secrets"}}, + } + return k8sClient.Update(context.TODO(), clusterRole) + }) + Expect(err).To(BeNil()) Byf("Create a ClusterProfile matching Cluster %s/%s", kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) clusterProfile := getClusterProfile(namePrefix, map[string]string{key: value}) @@ -116,37 +122,44 @@ var _ = Describe("Feature", func() { } Expect(k8sClient.Update(context.TODO(), currentConfigMap)).To(Succeed()) - Byf("Update ClusterProfile %s to reference both ConfigMaps", clusterProfile.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.TemplateResourceRefs = []configv1beta1.TemplateResourceRef{ - { - Resource: corev1.ObjectReference{ - Kind: "Secret", - Name: autoscaler, + + Byf("Update ClusterProfile %s to reference both ConfigMaps", clusterProfile.Name) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.TemplateResourceRefs = []configv1beta1.TemplateResourceRef{ + { + Resource: corev1.ObjectReference{ + Kind: "Secret", + Name: autoscaler, + }, + Identifier: "AutoscalerSecret", + Optional: true, }, - Identifier: "AutoscalerSecret", - Optional: true, - }, - } - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMapNs, - Name: configMap1Name, - // Deploy content of this configMap into management cluster - DeploymentType: configv1beta1.DeploymentTypeLocal, - }, - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMapNs, - Name: configMap2Name, - // Deploy content of this configMap into managed cluster - DeploymentType: configv1beta1.DeploymentTypeRemote, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + } + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMapNs, + Name: configMap1Name, + // Deploy content of this configMap into management cluster + DeploymentType: configv1beta1.DeploymentTypeLocal, + }, + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMapNs, + Name: configMap2Name, + // Deploy content of this configMap into managed cluster + DeploymentType: configv1beta1.DeploymentTypeRemote, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, clusterProfile.Name, ¤tClusterProfile.Spec, kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) diff --git a/test/fv/continue_on_error_test.go b/test/fv/continue_on_error_test.go index 71c6d697..894ee971 100644 --- a/test/fv/continue_on_error_test.go +++ b/test/fv/continue_on_error_test.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -105,57 +106,65 @@ var _ = Describe("Feature", Serial, func() { Byf("Update ClusterProfile %s to deploy helm charts and referencing ConfigMap %s/%s", clusterProfile.Name, configMap.Namespace, configMap.Name) + Byf("Setting ClusterProfile %s Spec.ContinueOnError true", clusterProfile.Name) + currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, - currentClusterProfile)).To(Succeed()) - // Cert-manager installation will fails as CRDs are not present and we are not deploying those - // ALso sets timeout otherwise helm takes too long before giving up on cert-manager failures (due to CRDs not being installed) - const two = 2 - helmTimeout := metav1.Duration{Duration: two * time.Minute} - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://charts.konghq.com", - RepositoryName: "kong", - ChartName: "kong/kong", - ChartVersion: "2.46.0", - ReleaseName: "kong", - ReleaseNamespace: "kong", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - { - RepositoryURL: "https://charts.jetstack.io", - RepositoryName: "jetstack", - ChartName: "jetstack/cert-manager", - ChartVersion: "v1.16.2", - ReleaseName: "cert-manager", - ReleaseNamespace: "cert-manager", - HelmChartAction: configv1beta1.HelmChartActionInstall, - Options: &configv1beta1.HelmOptions{ - Timeout: &helmTimeout, + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, + currentClusterProfile)).To(Succeed()) + + // Cert-manager installation will fails as CRDs are not present and we are not deploying those + // ALso sets timeout otherwise helm takes too long before giving up on cert-manager failures (due to CRDs not being installed) + const two = 2 + helmTimeout := metav1.Duration{Duration: two * time.Minute} + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://charts.konghq.com", + RepositoryName: "kong", + ChartName: "kong/kong", + ChartVersion: "2.46.0", + ReleaseName: "kong", + ReleaseNamespace: "kong", + HelmChartAction: configv1beta1.HelmChartActionInstall, }, - }, - { - RepositoryURL: "https://helm.nginx.com/stable/", - RepositoryName: "nginx-stable", - ChartName: "nginx-stable/nginx-ingress", - ChartVersion: "1.3.1", - ReleaseName: "nginx-latest", - ReleaseNamespace: "nginx", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - } - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: configMap.Name, - }, - } - Byf("Setting ClusterProfile %s Spec.ContinueOnError true", clusterProfile.Name) - currentClusterProfile.Spec.ContinueOnError = true + { + RepositoryURL: "https://charts.jetstack.io", + RepositoryName: "jetstack", + ChartName: "jetstack/cert-manager", + ChartVersion: "v1.16.2", + ReleaseName: "cert-manager", + ReleaseNamespace: "cert-manager", + HelmChartAction: configv1beta1.HelmChartActionInstall, + Options: &configv1beta1.HelmOptions{ + Timeout: &helmTimeout, + }, + }, + { + RepositoryURL: "https://helm.nginx.com/stable/", + RepositoryName: "nginx-stable", + ChartName: "nginx-stable/nginx-ingress", + ChartVersion: "1.3.1", + ReleaseName: "nginx-latest", + ReleaseNamespace: "nginx", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + } + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + } + currentClusterProfile.Spec.ContinueOnError = true + + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, + currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/dependencies_test.go b/test/fv/dependencies_test.go index 12a365c4..7004aa2e 100644 --- a/test/fv/dependencies_test.go +++ b/test/fv/dependencies_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -53,45 +54,61 @@ var _ = Describe("Dependencies", func() { Byf("Update ClusterProfile %s to deploy helm charts", clusterProfileDependency.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfileDependency.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://airflow.apache.org", - RepositoryName: "apache-airflow", - ChartName: "apache-airflow/airflow", - ChartVersion: "1.15.0", - ReleaseName: "airflow", - ReleaseNamespace: "airflow", - HelmChartAction: configv1beta1.HelmChartActionInstall, - Values: `createUserJob: + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfileDependency.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://airflow.apache.org", + RepositoryName: "apache-airflow", + ChartName: "apache-airflow/airflow", + ChartVersion: "1.15.0", + ReleaseName: "airflow", + ReleaseNamespace: "airflow", + HelmChartAction: configv1beta1.HelmChartActionInstall, + Values: `createUserJob: useHelmHooks: false applyCustomEnv: false migrateDatabaseJob: useHelmHooks: false applyCustomEnv: false`, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfileDependency.Name}, currentClusterProfile)).To(Succeed()) clusterSummaryDependency := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) Byf("Update ClusterProfile %s to deploy helm charts", clusterProfile.Name) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://charts.bitnami.com/bitnami", - RepositoryName: "bitnami", - ChartName: "bitnami/flink", - ChartVersion: "1.4.0", - ReleaseName: "flink", - ReleaseNamespace: "flink", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - } - - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://charts.bitnami.com/bitnami", + RepositoryName: "bitnami", + ChartName: "bitnami/flink", + ChartVersion: "1.4.0", + ReleaseName: "flink", + ReleaseNamespace: "flink", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + } + + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) diff --git a/test/fv/drift_test.go b/test/fv/drift_test.go index 9619b68b..8bfa1453 100644 --- a/test/fv/drift_test.go +++ b/test/fv/drift_test.go @@ -31,6 +31,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -148,20 +149,24 @@ var _ = Describe("Helm", Serial, func() { Expect(k8sClient.Create(context.TODO(), labelsConfigMap)).To(Succeed()) Byf("Update ClusterProfile %s to deploy helm charts", clusterProfile.Name) + By("use driftExclusion to ignore cleanup controller spec/replicas changes") + By("Use patches to add projectsveltos.io/driftDetectionIgnore annotation") + currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), - types.NamespacedName{Name: clusterProfile.Name}, - currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://kyverno.github.io/kyverno/", - RepositoryName: "kyverno", - ChartName: "kyverno/kyverno", - ChartVersion: "v3.2.6", - ReleaseName: "kyverno-latest", - ReleaseNamespace: "kyverno", - HelmChartAction: configv1beta1.HelmChartActionInstall, - Values: `admissionController: + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://kyverno.github.io/kyverno/", + RepositoryName: "kyverno", + ChartName: "kyverno/kyverno", + ChartVersion: "v3.2.6", + ReleaseName: "kyverno-latest", + ReleaseNamespace: "kyverno", + HelmChartAction: configv1beta1.HelmChartActionInstall, + Values: `admissionController: replicas: 1 backgroundController: replicas: 1 @@ -169,57 +174,60 @@ cleanupController: replicas: 1 reportsController: replicas: 1`, - ValuesFrom: []configv1beta1.ValueFrom{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: labelsConfigMap.Namespace, - Name: labelsConfigMap.Name, - }, - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: cleanupControllerConfigMap.Namespace, - Name: cleanupControllerConfigMap.Name, - }, - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: admissionControllerConfigMap.Namespace, - Name: admissionControllerConfigMap.Name, + ValuesFrom: []configv1beta1.ValueFrom{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: labelsConfigMap.Namespace, + Name: labelsConfigMap.Name, + }, + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: cleanupControllerConfigMap.Namespace, + Name: cleanupControllerConfigMap.Name, + }, + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: admissionControllerConfigMap.Namespace, + Name: admissionControllerConfigMap.Name, + }, }, }, - }, - } + } - By("use driftExclusion to ignore cleanup controller spec/replicas changes") - currentClusterProfile.Spec.DriftExclusions = []configv1beta1.DriftExclusion{ - { - Paths: []string{"/spec/replicas"}, - Target: &libsveltosv1beta1.PatchSelector{ - Kind: "Deployment", - Group: "apps", - Version: "v1", - Namespace: kyvernoNamespace, - Name: cleanupControllerDeplName, + currentClusterProfile.Spec.DriftExclusions = []configv1beta1.DriftExclusion{ + { + Paths: []string{"/spec/replicas"}, + Target: &libsveltosv1beta1.PatchSelector{ + Kind: "Deployment", + Group: "apps", + Version: "v1", + Namespace: kyvernoNamespace, + Name: cleanupControllerDeplName, + }, }, - }, - } + } - By("Use patches to add projectsveltos.io/driftDetectionIgnore annotation") - currentClusterProfile.Spec.Patches = []libsveltosv1beta1.Patch{ - { - Patch: `- op: add + currentClusterProfile.Spec.Patches = []libsveltosv1beta1.Patch{ + { + Patch: `- op: add path: /metadata/annotations/projectsveltos.io~1driftDetectionIgnore value: ok`, - Target: &libsveltosv1beta1.PatchSelector{ - Group: "apps", - Version: "v1", - Kind: "Deployment", - Namespace: kyvernoNamespace, - Name: admissionControllerDeplName, + Target: &libsveltosv1beta1.PatchSelector{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: kyvernoNamespace, + Name: admissionControllerDeplName, + }, }, - }, - } + } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, @@ -543,29 +551,67 @@ func verifyResourceSummary(c client.Client, clusterSummary *configv1beta1.Cluste Expect(currentResourceSummary).ToNot(BeNil()) deploymentKind := "Deployment" + verifyAdmissionControllerDeployment(c, currentResourceSummary, deploymentKind) + + verifySpecReplicas(currentResourceSummary, deploymentKind) +} + +func verifyAdmissionControllerDeployment(c client.Client, + currentResourceSummary *libsveltosv1beta1.ResourceSummary, deploymentKind string) { // Patches has been configured to ignore admission controller for configuration // drift (by adding annotation projectsveltos.io/driftDetectionIgnore) Byf("Verify deployment %s/%s is marked to be ignored for configuration drift", kyvernoNamespace, admissionControllerDeplName) - found := false + Eventually(func() bool { + resourceSummaries := &libsveltosv1beta1.ResourceSummaryList{} + err := c.List(context.TODO(), resourceSummaries) + if err != nil { + return false + } + if len(resourceSummaries.Items) != 1 { + return false + } + + currentResourceSummary := &resourceSummaries.Items[0] + + found := false + ignore := false + for i := range currentResourceSummary.Spec.ChartResources { + for j := range currentResourceSummary.Spec.ChartResources[i].Resources { + r := ¤tResourceSummary.Spec.ChartResources[i].Resources[j] + if r.Kind == deploymentKind && r.Namespace == kyvernoNamespace && + r.Name == admissionControllerDeplName { + + ignore = r.IgnoreForConfigurationDrift + found = true + break + } + } + } + return found && ignore + }, timeout, pollingInterval).Should(BeTrue()) + + // DriftExclusion has been configured to NOT ignore for anything else for i := range currentResourceSummary.Spec.ChartResources { for j := range currentResourceSummary.Spec.ChartResources[i].Resources { r := ¤tResourceSummary.Spec.ChartResources[i].Resources[j] if r.Kind == deploymentKind && r.Namespace == kyvernoNamespace && r.Name == admissionControllerDeplName { - Expect(r.IgnoreForConfigurationDrift).To(BeTrue()) - found = true + continue } else { Expect(r.IgnoreForConfigurationDrift).To(BeFalse()) } } } - Expect(found).To(BeTrue()) +} +func verifySpecReplicas(currentResourceSummary *libsveltosv1beta1.ResourceSummary, + deploymentKind string) { + + found := false // DriftExclusion has been configured to ignore cleanup controller spec/replicas - found = false Byf("Verify deployment %s/%s spec/replicas is marked to be ignored for configuration drift", kyvernoNamespace, cleanupControllerDeplName) for i := range currentResourceSummary.Spec.Patches { diff --git a/test/fv/dryrun_test.go b/test/fv/dryrun_test.go index 5275fdf9..0fc15e0d 100644 --- a/test/fv/dryrun_test.go +++ b/test/fv/dryrun_test.go @@ -26,6 +26,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" @@ -103,30 +104,42 @@ var _ = Describe("DryRun", func() { Byf("Update ClusterProfile %s to reference ConfigMap with Kong ServiceAccount %s/%s", clusterProfile.Name, kongSAConfigMap.Namespace, kongSAConfigMap.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: kongSAConfigMap.Namespace, - Name: kongSAConfigMap.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: kongSAConfigMap.Namespace, + Name: kongSAConfigMap.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) Byf("Update ClusterProfile %s to deploy mysql helm chart", clusterProfile.Name) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://helm.mariadb.com/mariadb-operator", - RepositoryName: "mariadb-operator", - ChartName: "mariadb-operator/mariadb-operator", - ChartVersion: "0.35.1", - ReleaseName: "mariadb", - ReleaseNamespace: "mariadb", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://helm.mariadb.com/mariadb-operator", + RepositoryName: "mariadb-operator", + ChartName: "mariadb-operator/mariadb-operator", + ChartVersion: "0.35.1", + ReleaseName: "mariadb", + ReleaseNamespace: "mariadb", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, @@ -171,57 +184,63 @@ var _ = Describe("DryRun", func() { kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) Byf("Update ClusterProfile %s to reference configMaps with Kong's configuration", dryRunClusterProfile.Name) - Expect(k8sClient.Get(context.TODO(), - types.NamespacedName{Name: dryRunClusterProfile.Name}, - currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMapNs, Name: kongRoleConfigMap.Name, - }, - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMapNs, Name: kongSAConfigMap.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: dryRunClusterProfile.Name}, + currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMapNs, Name: kongRoleConfigMap.Name, + }, + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMapNs, Name: kongSAConfigMap.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) Byf("Update ClusterProfile %s to reference some helm charts", dryRunClusterProfile.Name) - Expect(k8sClient.Get(context.TODO(), - types.NamespacedName{Name: dryRunClusterProfile.Name}, - currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://helm.mariadb.com/mariadb-operator", - RepositoryName: "mariadb-operator", - ChartName: "mariadb-operator/mariadb-operator", - ChartVersion: "0.36.0", - ReleaseName: "mariadb", - ReleaseNamespace: "mariadb", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - { - RepositoryURL: "https://charts.jetstack.io", - RepositoryName: "jetstack", - ChartName: "jetstack/cert-manager", - ChartVersion: "v1.16.2", - ReleaseName: "cert-manager", - ReleaseNamespace: "cert-manager", - HelmChartAction: configv1beta1.HelmChartActionInstall, - Values: `crds: + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: dryRunClusterProfile.Name}, + currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://helm.mariadb.com/mariadb-operator", + RepositoryName: "mariadb-operator", + ChartName: "mariadb-operator/mariadb-operator", + ChartVersion: "0.36.0", + ReleaseName: "mariadb", + ReleaseNamespace: "mariadb", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + { + RepositoryURL: "https://charts.jetstack.io", + RepositoryName: "jetstack", + ChartName: "jetstack/cert-manager", + ChartVersion: "v1.16.2", + ReleaseName: "cert-manager", + ReleaseNamespace: "cert-manager", + HelmChartAction: configv1beta1.HelmChartActionInstall, + Values: `crds: enabled: true`, - }, - { - RepositoryURL: "https://cloudnative-pg.github.io/charts", - RepositoryName: "cloudnative-pg", - ChartName: "cloudnative-pg/cloudnative-pg", - ChartVersion: "0.22.1", - ReleaseName: "cnpg", - ReleaseNamespace: "cnpg-system", - HelmChartAction: configv1beta1.HelmChartActionUninstall, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + }, + { + RepositoryURL: "https://cloudnative-pg.github.io/charts", + RepositoryName: "cloudnative-pg", + ChartName: "cloudnative-pg/cloudnative-pg", + ChartVersion: "0.22.1", + ReleaseName: "cnpg", + ReleaseNamespace: "cnpg-system", + HelmChartAction: configv1beta1.HelmChartActionUninstall, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) dryRunClusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/helm_options_test.go b/test/fv/helm_options_test.go index 512d21c5..3697c55e 100644 --- a/test/fv/helm_options_test.go +++ b/test/fv/helm_options_test.go @@ -26,6 +26,7 @@ import ( appsv1 "k8s.io/api/apps/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -49,26 +50,34 @@ var _ = Describe("HelmOptions", func() { Byf("Update ClusterProfile %s to deploy helm charts", clusterProfile.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://kubernetes-sigs.github.io/external-dns/", - RepositoryName: "external-dns", - ChartName: "external-dns/external-dns", - ChartVersion: "1.14.5", - ReleaseName: "external-dns", - ReleaseNamespace: "external-dns", - HelmChartAction: configv1beta1.HelmChartActionInstall, - Options: &configv1beta1.HelmOptions{ - DependencyUpdate: true, - InstallOptions: configv1beta1.HelmInstallOptions{ - Replace: false, + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://kubernetes-sigs.github.io/external-dns/", + RepositoryName: "external-dns", + ChartName: "external-dns/external-dns", + ChartVersion: "1.14.5", + ReleaseName: "external-dns", + ReleaseNamespace: "external-dns", + HelmChartAction: configv1beta1.HelmChartActionInstall, + Options: &configv1beta1.HelmOptions{ + DependencyUpdate: true, + InstallOptions: configv1beta1.HelmInstallOptions{ + Replace: false, + }, }, }, - }, - } + } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, @@ -102,27 +111,35 @@ var _ = Describe("HelmOptions", func() { nil, charts) Byf("Update ClusterProfile %s to upgrade external-dns helm charts", clusterProfile.Name) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://kubernetes-sigs.github.io/external-dns/", - RepositoryName: "external-dns", - ChartName: "external-dns/external-dns", - ChartVersion: "1.14.4", - ReleaseName: "external-dns", - ReleaseNamespace: "external-dns", - HelmChartAction: configv1beta1.HelmChartActionInstall, - Options: &configv1beta1.HelmOptions{ - DependencyUpdate: true, - UpgradeOptions: configv1beta1.HelmUpgradeOptions{ - ResetValues: false, - MaxHistory: 5, + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://kubernetes-sigs.github.io/external-dns/", + RepositoryName: "external-dns", + ChartName: "external-dns/external-dns", + ChartVersion: "1.14.4", + ReleaseName: "external-dns", + ReleaseNamespace: "external-dns", + HelmChartAction: configv1beta1.HelmChartActionInstall, + Options: &configv1beta1.HelmOptions{ + DependencyUpdate: true, + UpgradeOptions: configv1beta1.HelmUpgradeOptions{ + ResetValues: false, + MaxHistory: 5, + }, }, }, - }, - } + } + + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, @@ -143,27 +160,32 @@ var _ = Describe("HelmOptions", func() { }, timeout, pollingInterval).Should(BeTrue()) Byf("Update ClusterProfile %s to uninstall external-dns helm charts", clusterProfile.Name) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://kubernetes-sigs.github.io/external-dns/", - RepositoryName: "external-dns", - ChartName: "external-dns/external-dns", - ChartVersion: "1.14.4", - ReleaseName: "external-dns", - ReleaseNamespace: "external-dns", - HelmChartAction: configv1beta1.HelmChartActionUninstall, - Options: &configv1beta1.HelmOptions{ - DependencyUpdate: true, - UninstallOptions: configv1beta1.HelmUninstallOptions{ - KeepHistory: true, - DeletionPropagation: "background", + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://kubernetes-sigs.github.io/external-dns/", + RepositoryName: "external-dns", + ChartName: "external-dns/external-dns", + ChartVersion: "1.14.4", + ReleaseName: "external-dns", + ReleaseNamespace: "external-dns", + HelmChartAction: configv1beta1.HelmChartActionUninstall, + Options: &configv1beta1.HelmOptions{ + DependencyUpdate: true, + UninstallOptions: configv1beta1.HelmUninstallOptions{ + KeepHistory: true, + DeletionPropagation: "background", + }, }, }, - }, - } + } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) Byf("Verifying external-dns deployment is removed from workload cluster") Eventually(func() bool { diff --git a/test/fv/helm_test.go b/test/fv/helm_test.go index c7108b4d..91325172 100644 --- a/test/fv/helm_test.go +++ b/test/fv/helm_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -63,42 +64,50 @@ var _ = Describe("Helm", func() { clusterProfile.Name, &clusterProfile.Spec, kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) Byf("Update ClusterProfile %s to deploy helm charts", clusterProfile.Name) - currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://kyverno.github.io/kyverno/", - RepositoryName: "kyverno", - ChartName: "kyverno/kyverno", - ChartVersion: "v3.3.4", - ReleaseName: "kyverno-latest", - ReleaseNamespace: "kyverno", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - { - RepositoryURL: "https://docs.wildfly.org/wildfly-charts/", - RepositoryName: "wildfly", - ChartName: "wildfly/wildfly", - ChartVersion: "2.4.0", - ReleaseName: "wildfly", - ReleaseNamespace: "wildfly", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - } - currentClusterProfile.Spec.ValidateHealths = []configv1beta1.ValidateHealth{ - { - Name: "kyverno-deployment-health", - FeatureID: configv1beta1.FeatureHelm, - Namespace: "kyverno", - Group: "apps", - Version: "v1", - Kind: "Deployment", - Script: luaEvaluateDeploymentHealth, - }, - } + currentClusterProfile := &configv1beta1.ClusterProfile{} + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://kyverno.github.io/kyverno/", + RepositoryName: "kyverno", + ChartName: "kyverno/kyverno", + ChartVersion: "v3.3.4", + ReleaseName: "kyverno-latest", + ReleaseNamespace: "kyverno", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + { + RepositoryURL: "https://docs.wildfly.org/wildfly-charts/", + RepositoryName: "wildfly", + ChartName: "wildfly/wildfly", + ChartVersion: "2.4.0", + ReleaseName: "wildfly", + ReleaseNamespace: "wildfly", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + } + + currentClusterProfile.Spec.ValidateHealths = []configv1beta1.ValidateHealth{ + { + Name: "kyverno-deployment-health", + FeatureID: configv1beta1.FeatureHelm, + Namespace: "kyverno", + Group: "apps", + Version: "v1", + Kind: "Deployment", + Script: luaEvaluateDeploymentHealth, + }, + } + + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/helm_with_corev1list_test.go b/test/fv/helm_with_corev1list_test.go index 9d152886..f6d7d17a 100644 --- a/test/fv/helm_with_corev1list_test.go +++ b/test/fv/helm_with_corev1list_test.go @@ -74,9 +74,9 @@ var _ = Describe("Feature", func() { // This Helm release contains corev1.List. Verify those are expanded so drift-detection can watch for configuration drift It("Expand corev1List in helm charts", Label("NEW-FV"), func() { - Byf("Create a ClusterProfile matching Cluster %s/%s", kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) + Byf("Get a ClusterProfile matching Cluster %s/%s", kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) clusterProfile := getClusterProfile(namePrefix, map[string]string{key: value}) - Byf("Update ClusterProfile %s to deploy kube-prometheus-stack helm chart", clusterProfile.Name) + Byf("Create ClusterProfile %s to deploy kube-prometheus-stack helm chart", clusterProfile.Name) clusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ { RepositoryURL: "https://prometheus-community.github.io/helm-charts", diff --git a/test/fv/kustomize_test.go b/test/fv/kustomize_test.go index fb740fbe..eeebf4ae 100644 --- a/test/fv/kustomize_test.go +++ b/test/fv/kustomize_test.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -61,17 +62,24 @@ var _ = Describe("Kustomize with GitRepository", func() { Byf("Update ClusterProfile %s to reference GitRepository flux-system/flux-system", clusterProfile.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.KustomizationRefs = []configv1beta1.KustomizationRef{ - { - Kind: sourcev1.GitRepositoryKind, - Namespace: gitRepositoryNamespace, - Name: gitRepositoryName, - Path: "./helloWorld", - TargetNamespace: targetNamespace, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.KustomizationRefs = []configv1beta1.KustomizationRef{ + { + Kind: sourcev1.GitRepositoryKind, + Namespace: gitRepositoryNamespace, + Name: gitRepositoryName, + Path: "./helloWorld", + TargetNamespace: targetNamespace, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) Byf("Verifying GitRepository %s/%s exists", gitRepositoryNamespace, gitRepositoryName) gitRepository := &sourcev1.GitRepository{} diff --git a/test/fv/kustomize_with_configmap_test.go b/test/fv/kustomize_with_configmap_test.go index 78adf689..75a2a3f3 100644 --- a/test/fv/kustomize_with_configmap_test.go +++ b/test/fv/kustomize_with_configmap_test.go @@ -28,6 +28,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -87,24 +88,31 @@ var _ = Describe("Kustomize with ConfigMap", func() { Byf("Update ClusterProfile %s to reference ConfigMap kustomize", clusterProfile.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.KustomizationRefs = []configv1beta1.KustomizationRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: defaultNamespace, - Name: kustomizeConfigMapName, // this is created by Makefile and contains kustomize files - Path: "./overlays/production/", - TargetNamespace: targetNamespace, - ValuesFrom: []configv1beta1.ValueFrom{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: labelsConfigMap.Namespace, - Name: labelsConfigMap.Name, + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.KustomizationRefs = []configv1beta1.KustomizationRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: defaultNamespace, + Name: kustomizeConfigMapName, // this is created by Makefile and contains kustomize files + Path: "./overlays/production/", + TargetNamespace: targetNamespace, + ValuesFrom: []configv1beta1.ValueFrom{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: labelsConfigMap.Namespace, + Name: labelsConfigMap.Name, + }, }, }, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/leave_helm_charts_test.go b/test/fv/leave_helm_charts_test.go index b8835e5c..4d47c00e 100644 --- a/test/fv/leave_helm_charts_test.go +++ b/test/fv/leave_helm_charts_test.go @@ -26,6 +26,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -58,19 +59,27 @@ var _ = Describe("Helm", func() { Byf("Update ClusterProfile %s to deploy helm charts", clusterProfile.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://operator.min.io", - RepositoryName: "minio-operator", - ChartName: "minio-operator/minio-operator", - ChartVersion: "4.3.7", - ReleaseName: minioReleaseName, - ReleaseNamespace: minioNamespace, - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://operator.min.io", + RepositoryName: "minio-operator", + ChartName: "minio-operator/minio-operator", + ChartVersion: "4.3.7", + ReleaseName: minioReleaseName, + ReleaseNamespace: minioNamespace, + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/leave_policies_test.go b/test/fv/leave_policies_test.go index e20663b4..226d2c88 100644 --- a/test/fv/leave_policies_test.go +++ b/test/fv/leave_policies_test.go @@ -28,6 +28,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -99,15 +100,23 @@ var _ = Describe("LeavePolicies", func() { Byf("Update ClusterProfile %s to reference ConfigMap %s/%s", clusterProfile.Name, configMap.Namespace, configMap.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: configMap.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/lua_instantiation_test.go b/test/fv/lua_instantiation_test.go index 78a0e1e1..a36651ea 100644 --- a/test/fv/lua_instantiation_test.go +++ b/test/fv/lua_instantiation_test.go @@ -26,6 +26,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -93,25 +94,33 @@ var _ = Describe("ConfigMap with Lua", func() { Byf("Update ClusterProfile %s to reference ConfigMap %s/%s", clusterProfile.Name, configMap.Namespace, configMap.Name) Byf("Update ClusterProfile %s to fetch Namespace %s", clusterProfile.Name, ns.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.TemplateResourceRefs = []configv1beta1.TemplateResourceRef{ - { - Identifier: "Namespace", // this is used in luaCode. Must match - Resource: corev1.ObjectReference{ - Kind: ns.Kind, - Name: ns.Name, - APIVersion: "v1", + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.TemplateResourceRefs = []configv1beta1.TemplateResourceRef{ + { + Identifier: "Namespace", // this is used in luaCode. Must match + Resource: corev1.ObjectReference{ + Kind: ns.Kind, + Name: ns.Name, + APIVersion: "v1", + }, }, - }, - } - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: configMap.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + } + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/missing_reference_test.go b/test/fv/missing_reference_test.go index 44966a08..df1255ed 100644 --- a/test/fv/missing_reference_test.go +++ b/test/fv/missing_reference_test.go @@ -26,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -61,15 +62,23 @@ var _ = Describe("Missing Reference", func() { Byf("Update ClusterProfile %s to reference ConfigMap %s/%s", clusterProfile.Name, configMapNamespace, configMapName) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMapNamespace, - Name: configMapName, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMapNamespace, + Name: configMapName, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) By("Verify ClusterSummary reports error about missing reference") clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, diff --git a/test/fv/oci_test.go b/test/fv/oci_test.go index 002bdcc2..854aa857 100644 --- a/test/fv/oci_test.go +++ b/test/fv/oci_test.go @@ -25,6 +25,7 @@ import ( appsv1 "k8s.io/api/apps/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -48,20 +49,28 @@ var _ = Describe("Helm", func() { Byf("Update ClusterProfile %s to deploy oci helm charts", clusterProfile.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "oci://registry-1.docker.io/bitnamicharts", - RepositoryName: "oci-vault", - ChartName: "vault", - ChartVersion: "1.6.0", - ReleaseName: "vault", - ReleaseNamespace: "vault", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "oci://registry-1.docker.io/bitnamicharts", + RepositoryName: "oci-vault", + ChartName: "vault", + ChartVersion: "1.6.0", + ReleaseName: "vault", + ReleaseNamespace: "vault", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + } + + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/optional_referenced_resource_test.go b/test/fv/optional_referenced_resource_test.go index 485e7957..efaabfde 100644 --- a/test/fv/optional_referenced_resource_test.go +++ b/test/fv/optional_referenced_resource_test.go @@ -26,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -78,44 +79,51 @@ metadata: Byf("Update ClusterProfile %s to deploy helm charts and referencing ConfigMap %s/%s", clusterProfile.Name, configMap.Namespace, configMap.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, - currentClusterProfile)).To(Succeed()) - - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://prometheus-community.github.io/helm-charts", - RepositoryName: "prometheus-community", - ChartName: "prometheus-community/prometheus", - ChartVersion: "25.24.0", - ReleaseName: "prometheus", - ReleaseNamespace: "prometheus", - HelmChartAction: configv1beta1.HelmChartActionInstall, - ValuesFrom: []configv1beta1.ValueFrom{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: namespace, - Name: randomString(), // this wont exist but it is marked as optional - Optional: true, + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://prometheus-community.github.io/helm-charts", + RepositoryName: "prometheus-community", + ChartName: "prometheus-community/prometheus", + ChartVersion: "25.24.0", + ReleaseName: "prometheus", + ReleaseNamespace: "prometheus", + HelmChartAction: configv1beta1.HelmChartActionInstall, + ValuesFrom: []configv1beta1.ValueFrom{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: namespace, + Name: randomString(), // this wont exist but it is marked as optional + Optional: true, + }, }, }, - }, - } + } - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: configMap.Name, - }, - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: randomString(), // this wont exist but it is marked as optional - Optional: true, - }, - } + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: randomString(), // this wont exist but it is marked as optional + Optional: true, + }, + } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/patch_multiple_resources_test.go b/test/fv/patch_multiple_resources_test.go index 76417adf..43e01d47 100644 --- a/test/fv/patch_multiple_resources_test.go +++ b/test/fv/patch_multiple_resources_test.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -58,27 +59,35 @@ var _ = Describe("Patch with multiple resources in ConfigMap", func() { Byf("Update ClusterProfile %s to reference ConfigMap %s/%s", clusterProfile.Name, cmNamespace, cmName) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: cm.Namespace, - Name: cm.Name, - }, - } - currentClusterProfile.Spec.Patches = []libsveltosv1beta1.Patch{ - { - Target: &libsveltosv1beta1.PatchSelector{ - Kind: "Namespace", - Name: ".*", + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: cm.Namespace, + Name: cm.Name, }, - Patch: `- op: add + } + currentClusterProfile.Spec.Patches = []libsveltosv1beta1.Patch{ + { + Target: &libsveltosv1beta1.PatchSelector{ + Kind: "Namespace", + Name: ".*", + }, + Patch: `- op: add path: /metadata/labels/type value: operator`, - }, - } + }, + } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/paused_test.go b/test/fv/paused_test.go index 6386004a..41b659d0 100644 --- a/test/fv/paused_test.go +++ b/test/fv/paused_test.go @@ -26,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" @@ -70,15 +71,23 @@ var _ = Describe("Feature", Serial, func() { Byf("Update ClusterProfile %s to reference ConfigMap %s/%s", clusterProfile.Name, configMap.Namespace, configMap.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: configMap.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/profile_local_test.go b/test/fv/profile_local_test.go index b84ceecc..cc2ecfcc 100644 --- a/test/fv/profile_local_test.go +++ b/test/fv/profile_local_test.go @@ -27,6 +27,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -48,16 +49,19 @@ metadata: It("deploymentType Local only allows resources to be created in the same namespace", Label("FV", "EXTENDED"), func() { By("Grant addon-controller permission to create/delete ServiceAccount in the management cluster") - clusterRole := &rbacv1.ClusterRole{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "addon-controller-role-extra"}, - clusterRole)).To(Succeed()) - clusterRole.Rules = append(clusterRole.Rules, - rbacv1.PolicyRule{ - Verbs: []string{"*"}, - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - }) - Expect(k8sClient.Update(context.TODO(), clusterRole)).To(Succeed()) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + clusterRole := &rbacv1.ClusterRole{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "addon-controller-role-extra"}, + clusterRole)).To(Succeed()) + clusterRole.Rules = append(clusterRole.Rules, + rbacv1.PolicyRule{ + Verbs: []string{"*"}, + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + }) + return k8sClient.Update(context.TODO(), clusterRole) + }) + Expect(err).To(BeNil()) Byf("Create a Profile matching Cluster %s/%s", kindWorkloadCluster.Namespace, kindWorkloadCluster.Name) diff --git a/test/fv/reloader_test.go b/test/fv/reloader_test.go index 540db0c2..ff4d8db8 100644 --- a/test/fv/reloader_test.go +++ b/test/fv/reloader_test.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -111,16 +112,23 @@ var _ = Describe("Reloader", func() { clusterProfile.Name, configMap.Namespace, configMap.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: configMap.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/resource_per_cluster_test.go b/test/fv/resource_per_cluster_test.go index b35442cc..75c56bf3 100644 --- a/test/fv/resource_per_cluster_test.go +++ b/test/fv/resource_per_cluster_test.go @@ -27,6 +27,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -71,20 +72,28 @@ var _ = Describe("Feature", func() { Byf("Update ClusterProfile %s to reference ConfigMap %s (namespace not set)", clusterProfile.Name, configMap.Name) Byf("Update ClusterProfile %s to reference Secret %s (namespace not set)", clusterProfile.Name, secret.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: "", - Name: configMap.Name, - }, - { - Kind: string(libsveltosv1beta1.SecretReferencedResourceKind), - Namespace: "", - Name: secret.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: "", + Name: configMap.Name, + }, + { + Kind: string(libsveltosv1beta1.SecretReferencedResourceKind), + Namespace: "", + Name: secret.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/resource_test.go b/test/fv/resource_test.go index 31bc235d..ab136fd3 100644 --- a/test/fv/resource_test.go +++ b/test/fv/resource_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -130,20 +131,28 @@ var _ = Describe("Feature", func() { Byf("Update ClusterProfile %s to reference ConfigMap %s/%s", clusterProfile.Name, configMap.Namespace, configMap.Name) Byf("Update ClusterProfile %s to reference Secret %s/%s", clusterProfile.Name, secret.Namespace, secret.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: configMap.Name, - }, - { - Kind: string(libsveltosv1beta1.SecretReferencedResourceKind), - Namespace: secret.Namespace, - Name: secret.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + { + Kind: string(libsveltosv1beta1.SecretReferencedResourceKind), + Namespace: secret.Namespace, + Name: secret.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/stale_resource_test.go b/test/fv/stale_resource_test.go index a772f4f5..9ef91494 100644 --- a/test/fv/stale_resource_test.go +++ b/test/fv/stale_resource_test.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -123,15 +124,23 @@ var _ = Describe("Stale Resources", func() { Byf("Update ClusterProfile %s to reference ConfigMap %s/%s", clusterProfile.Name, configMap.Namespace, configMap.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMap.Namespace, - Name: configMap.Name, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/tenant-admin_test.go b/test/fv/tenant-admin_test.go index 51ef19b5..f88cbe20 100644 --- a/test/fv/tenant-admin_test.go +++ b/test/fv/tenant-admin_test.go @@ -30,6 +30,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -151,21 +152,29 @@ var _ = Describe("Feature", func() { Byf("Update ClusterProfile %s to reference ConfigMap and add Tenant admin labels", clusterProfile.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Labels = map[string]string{ - libsveltosv1beta1.ServiceAccountNameLabel: tenantServiceAccount.Name, - libsveltosv1beta1.ServiceAccountNamespaceLabel: tenantServiceAccount.Namespace, - } - currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ - { - Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), - Namespace: configMapNs, - Name: configMapName, - // Deploy content of this configMap into management cluster - DeploymentType: configv1beta1.DeploymentTypeLocal, - }, - } - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Labels = map[string]string{ + libsveltosv1beta1.ServiceAccountNameLabel: tenantServiceAccount.Name, + libsveltosv1beta1.ServiceAccountNamespaceLabel: tenantServiceAccount.Namespace, + } + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMapNs, + Name: configMapName, + // Deploy content of this configMap into management cluster + DeploymentType: configv1beta1.DeploymentTypeLocal, + }, + } + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec, diff --git a/test/fv/tier_test.go b/test/fv/tier_test.go index 351804c0..5cea47c5 100644 --- a/test/fv/tier_test.go +++ b/test/fv/tier_test.go @@ -25,6 +25,7 @@ import ( appsv1 "k8s.io/api/apps/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" @@ -48,40 +49,48 @@ var _ = Describe("Helm", Serial, func() { Byf("Update ClusterProfile %s to deploy helm charts", clusterProfile.Name) currentClusterProfile := &configv1beta1.ClusterProfile{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) - currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ - { - RepositoryURL: "https://kyverno.github.io/kyverno/", - RepositoryName: "kyverno", - ChartName: "kyverno/kyverno", - ChartVersion: "v3.2.6", - ReleaseName: "kyverno-latest", - ReleaseNamespace: "kyverno", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - { - RepositoryURL: "https://prometheus-community.github.io/helm-charts", - RepositoryName: "prometheus-community", - ChartName: "prometheus-community/prometheus", - ChartVersion: "25.24.0", - ReleaseName: "prometheus", - ReleaseNamespace: "prometheus", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - { - RepositoryURL: "https://grafana.github.io/helm-charts", - RepositoryName: "grafana", - ChartName: "grafana/grafana", - ChartVersion: "8.3.4", - ReleaseName: "grafana", - ReleaseNamespace: "grafana", - HelmChartAction: configv1beta1.HelmChartActionInstall, - }, - } - currentClusterProfile.Spec.Tier = 90 - currentClusterProfile.Spec.ContinueOnConflict = true - Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.HelmCharts = []configv1beta1.HelmChart{ + { + RepositoryURL: "https://kyverno.github.io/kyverno/", + RepositoryName: "kyverno", + ChartName: "kyverno/kyverno", + ChartVersion: "v3.2.6", + ReleaseName: "kyverno-latest", + ReleaseNamespace: "kyverno", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + { + RepositoryURL: "https://prometheus-community.github.io/helm-charts", + RepositoryName: "prometheus-community", + ChartName: "prometheus-community/prometheus", + ChartVersion: "25.24.0", + ReleaseName: "prometheus", + ReleaseNamespace: "prometheus", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + { + RepositoryURL: "https://grafana.github.io/helm-charts", + RepositoryName: "grafana", + ChartName: "grafana/grafana", + ChartVersion: "8.3.4", + ReleaseName: "grafana", + ReleaseNamespace: "grafana", + HelmChartAction: configv1beta1.HelmChartActionInstall, + }, + } + currentClusterProfile.Spec.Tier = 90 + currentClusterProfile.Spec.ContinueOnConflict = true + + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) clusterSummary := verifyClusterSummary(controllers.ClusterProfileLabelName, currentClusterProfile.Name, ¤tClusterProfile.Spec,