CSPL-3775 Making some improvements

Author: kasiakoziol

test/monitoring_console/monitoring_console_test.go

@@ -19,6 +19,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3"
 	enterpriseApi "github.com/splunk/splunk-operator/api/v4"
@@ -39,7 +40,7 @@ var masterManagerMCConfigs = []testenv.MCVersionConfig{
 		DeployM4WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error {
 			return d.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, name, replicas, siteCount, mcRef, shc)
 		},
-		NewCMObject: func() interface{} { return &enterpriseApiV3.ClusterMaster{} },
+		NewCMObject: func() client.Object { return &enterpriseApiV3.ClusterMaster{} },
 		VerifyCMReady: func(ctx context.Context, d *testenv.Deployment, te *testenv.TestCaseEnv) error {
 			return te.VerifyClusterMasterReady(ctx, d)
 		},
@@ -56,7 +57,7 @@ var masterManagerMCConfigs = []testenv.MCVersionConfig{
 		DeployM4WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error {
 			return d.DeployMultisiteClusterWithMonitoringConsole(ctx, name, replicas, siteCount, mcRef, shc)
 		},
-		NewCMObject: func() interface{} { return &enterpriseApi.ClusterManager{} },
+		NewCMObject: func() client.Object { return &enterpriseApi.ClusterManager{} },
 		VerifyCMReady: func(ctx context.Context, d *testenv.Deployment, te *testenv.TestCaseEnv) error {
 			return te.VerifyClusterManagerReady(ctx, d)
 		},

test/smartstore/smartstore_test_shared.go

@@ -28,6 +28,7 @@ import (
 // RunS1MultipleIndexesTest runs the standard S1 multiple indexes SmartStore test workflow
 func RunS1MultipleIndexesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, waitTimeout time.Duration) {
 	volName := "test-volume-" + testenv.RandomDNSName(3)
+	// Each key is unique at runtime because RandomDNSName generates distinct suffixes.
 	indexVolumeMap := map[string]string{
 		"test-index-" + testenv.RandomDNSName(3): volName,
 		"test-index-" + testenv.RandomDNSName(3): volName,
@@ -128,19 +129,19 @@ func RunS1DefaultVolumesTest(ctx context.Context, deployment *testenv.Deployment
 	serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf"
 
 	// Validate MaxCacheSizeMB
-	Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB))).To(Succeed(), "MaxCacheSizeMB mismatch")
+	Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB))).To(Succeed(), "MaxCacheSizeMB mismatch")
 
 	// Validate EvictionPaddingSizeMB
-	Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB))).To(Succeed(), "EvictionPaddingSizeMB mismatch")
+	Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB))).To(Succeed(), "EvictionPaddingSizeMB mismatch")
 
 	// Validate MaxConcurrentDownloads
-	Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads))).To(Succeed(), "MaxConcurrentDownloads mismatch")
+	Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads))).To(Succeed(), "MaxConcurrentDownloads mismatch")
 
 	// Validate MaxConcurrentUploads
-	Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads))).To(Succeed(), "MaxConcurrentUploads mismatch")
+	Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads))).To(Succeed(), "MaxConcurrentUploads mismatch")
 
 	// Validate EvictionPolicy
-	Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy)).To(Succeed(), "EvictionPolicy mismatch")
+	Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy)).To(Succeed(), "EvictionPolicy mismatch")
 }
 
 // RunS1EphemeralStorageTest deploys a Standalone with one ephemeral storage volume configured and verifies it is ready.

test/testenv/appframework_utils.go

@@ -490,7 +490,9 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym
 				verifyCtx, cancel := context.WithTimeout(ctx, deployment.GetTimeout())
 				defer cancel()
 
-				testenv.VerifyAppListPhase(verifyCtx, deployment, as.CrName, as.CrKind, as.CrAppSourceName, p, as.CrAppFileList)
+				if err := testenv.VerifyAppListPhase(verifyCtx, deployment, as.CrName, as.CrKind, as.CrAppSourceName, p, as.CrAppFileList); err != nil {
+					errChan <- fmt.Errorf("phase %v verification failed for CR %s/%s: %w", p, as.CrKind, as.CrName, err)
+				}
 			}(appSourceItem, phase)
 		}
 	}
@@ -515,7 +517,9 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym
 	for _, appSource := range appSource {
 		testenv.Log.Info(fmt.Sprintf("Verify apps %s packages are deleted from the operator pod for CR %v with name %v", appSource.CrAppVersion, appSource.CrKind, appSource.CrName))
 		opPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenv.GetName(), appSource.CrKind, deployment.GetName(), appSource.CrAppScope, appSource.CrAppSourceName)
-		testenv.VerifyAppsPackageDeletedOnOperatorContainer(ctx, deployment, []string{opPod}, appSource.CrAppFileList, opPath)
+		if err := testenv.VerifyAppsPackageDeletedOnOperatorContainer(ctx, deployment, []string{opPod}, appSource.CrAppFileList, opPath); err != nil {
+			return "", fmt.Errorf("apps packages not deleted from operator pod for CR %s/%s: %w", appSource.CrKind, appSource.CrName, err)
+		}
 	}
 
 	// Verify apps 'install' state for all CRs IN PARALLEL
@@ -537,7 +541,9 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym
 			verifyCtx, cancel := context.WithTimeout(ctx, deployment.GetTimeout())
 			defer cancel()
 
-			testenv.VerifyAppListPhase(verifyCtx, deployment, as.CrName, as.CrKind, as.CrAppSourceName, enterpriseApi.PhaseInstall, as.CrAppFileList)
+			if err := testenv.VerifyAppListPhase(verifyCtx, deployment, as.CrName, as.CrKind, as.CrAppSourceName, enterpriseApi.PhaseInstall, as.CrAppFileList); err != nil {
+				errChan <- fmt.Errorf("install phase verification failed for CR %s/%s: %w", as.CrKind, as.CrName, err)
+			}
 		}(appSourceItem)
 	}
 
@@ -561,34 +567,46 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym
 		podDownloadPath := AppStagingLocOnPod + appSource.CrAppSourceVolumeName
 		pod := appSource.CrPod
 		testenv.Log.Info(fmt.Sprintf("Verify %s apps packages are deleted on pod %s", appSource.CrAppVersion, pod))
-		testenv.VerifyAppsPackageDeletedOnContainer(ctx, deployment, pod, appSource.CrAppFileList, podDownloadPath)
+		if err := testenv.VerifyAppsPackageDeletedOnContainer(ctx, deployment, pod, appSource.CrAppFileList, podDownloadPath); err != nil {
+			return "", fmt.Errorf("apps packages not deleted on pod %s for CR %s/%s: %w", pod, appSource.CrKind, appSource.CrName, err)
+		}
 	}
 
 	// Verify bundle push status
 	for _, appSource := range appSource {
 		if (appSource.CrKind == "ClusterManager" || appSource.CrKind == "ClusterMaster") && appSource.CrAppScope == enterpriseApi.ScopeCluster {
 			testenv.Log.Info(fmt.Sprintf("Verify Cluster Manager bundle push status (%s apps) and compare bundle hash with previous bundle hash", appSource.CrAppVersion))
-			testenv.VerifyClusterManagerBundlePush(ctx, deployment, appSource.CrReplicas, clusterManagerBundleHash)
+			if err := testenv.VerifyClusterManagerBundlePush(ctx, deployment, appSource.CrReplicas, clusterManagerBundleHash); err != nil {
+				return "", fmt.Errorf("cluster manager bundle push verification failed: %w", err)
+			}
 			if clusterManagerBundleHash == "" {
 				clusterManagerBundleHash = GetClusterManagerBundleHash(ctx, deployment, appSource.CrKind)
 			}
 		}
 		if appSource.CrKind == "SearchHeadCluster" && appSource.CrAppScope == enterpriseApi.ScopeCluster {
 			testenv.Log.Info(fmt.Sprintf("Verify Deployer bundle push status (%s apps)", appSource.CrAppVersion))
-			testenv.VerifyDeployerBundlePush(ctx, deployment, testenv.GetName(), appSource.CrReplicas)
+			if err := testenv.VerifyDeployerBundlePush(ctx, deployment, testenv.GetName(), appSource.CrReplicas); err != nil {
+				return "", fmt.Errorf("deployer bundle push verification failed: %w", err)
+			}
 		}
 	}
 
 	// Verify apps are copied to correct location on all CRs
 	for _, appSource := range appSource {
 		if appSource.CrAppScope == enterpriseApi.ScopeLocal {
 			testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'local' scope are copied to /etc/apps/ for CR %s with name %s", appSource.CrAppVersion, appSource.CrKind, appSource.CrName))
-			testenv.VerifyAppsCopied(ctx, deployment, appSource.CrPod, appSource.CrAppList, true, appSource.CrAppScope)
+			if err := testenv.VerifyAppsCopied(ctx, deployment, appSource.CrPod, appSource.CrAppList, true, appSource.CrAppScope); err != nil {
+				return "", fmt.Errorf("local apps not copied for CR %s/%s: %w", appSource.CrKind, appSource.CrName, err)
+			}
 		} else {
 			testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'cluster' scope are NOT copied to /etc/apps/ on %v pod", appSource.CrAppVersion, appSource.CrPod))
-			testenv.VerifyAppsCopied(ctx, deployment, appSource.CrPod, appSource.CrAppList, false, appSource.CrAppScope)
+			if err := testenv.VerifyAppsCopied(ctx, deployment, appSource.CrPod, appSource.CrAppList, false, appSource.CrAppScope); err != nil {
+				return "", fmt.Errorf("cluster apps unexpectedly copied to CR pod for %s/%s: %w", appSource.CrKind, appSource.CrName, err)
+			}
 			testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'cluster' scope are copied on %v pods", appSource.CrAppVersion, appSource.CrClusterPods))
-			testenv.VerifyAppsCopied(ctx, deployment, appSource.CrClusterPods, appSource.CrAppList, true, appSource.CrAppScope)
+			if err := testenv.VerifyAppsCopied(ctx, deployment, appSource.CrClusterPods, appSource.CrAppList, true, appSource.CrAppScope); err != nil {
+				return "", fmt.Errorf("cluster apps not copied to cluster pods for %s/%s: %w", appSource.CrKind, appSource.CrName, err)
+			}
 		}
 	}
 
@@ -598,11 +616,15 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym
 		checkUpdated := appSource.CrAppVersion == "V2"
 		if appSource.CrAppScope == "local" {
 			testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'local' scope for CR %s with name %s are installed on pod %s", appSource.CrAppVersion, appSource.CrKind, appSource.CrName, allPodNames))
-			testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, false)
+			if err := testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, false); err != nil {
+				return "", fmt.Errorf("local apps not installed for CR %s/%s: %w", appSource.CrKind, appSource.CrName, err)
+			}
 		} else {
 			allPodNames = appSource.CrClusterPods
 			testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'cluster' scope for CR %s with name %s are installed on pods %s", appSource.CrAppVersion, appSource.CrKind, appSource.CrName, allPodNames))
-			testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, true)
+			if err := testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, true); err != nil {
+				return "", fmt.Errorf("cluster apps not installed for CR %s/%s: %w", appSource.CrKind, appSource.CrName, err)
+			}
 		}
 	}
 	return clusterManagerBundleHash, nil

test/testenv/assertion_helpers.go

@@ -21,6 +21,7 @@ import (
 	enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3"
 	enterpriseApi "github.com/splunk/splunk-operator/api/v4"
 	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // ScaleSearchHeadCluster scales a Search Head Cluster to the specified replica count
@@ -64,7 +65,7 @@ func (testcaseenv *TestCaseEnv) ScaleIndexerCluster(ctx context.Context, deploym
 }
 
 // UpdateMonitoringConsoleRefAndVerify updates the MonitoringConsoleRef in a CR and waits for the change to apply
-func (testcaseenv *TestCaseEnv) UpdateMonitoringConsoleRefAndVerify(ctx context.Context, deployment *Deployment, obj interface{}, instanceName string, newMCName string) error {
+func (testcaseenv *TestCaseEnv) UpdateMonitoringConsoleRefAndVerify(ctx context.Context, deployment *Deployment, obj client.Object, instanceName string, newMCName string) error {
 	// Get current resource version before update
 	resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, obj)
 
@@ -157,17 +158,11 @@ func (testcaseenv *TestCaseEnv) VerifyM1ClusterReady(ctx context.Context, deploy
 }
 
 // VerifyM4ClusterReady verifies the cluster coordinator, indexers, multisite status, and SHC are ready.
+// It delegates to VerifyM4ComponentsReady, adapting the callback signature.
 func (testcaseenv *TestCaseEnv) VerifyM4ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment) error) error {
-	if err := verifyCoordinator(ctx, deployment); err != nil {
-		return err
-	}
-	if err := testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount); err != nil {
-		return err
-	}
-	if err := testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount); err != nil {
-		return err
-	}
-	return testcaseenv.VerifySearchHeadClusterReady(ctx, deployment)
+	return testcaseenv.VerifyM4ComponentsReady(ctx, deployment, siteCount, func() error {
+		return verifyCoordinator(ctx, deployment)
+	})
 }
 
 // VerifyM4IndexersAndSHCReady verifies the cluster coordinator, indexers, and SHC are ready (without multisite check).

test/testenv/cmutil.go

@@ -33,7 +33,7 @@ type ClusterManagerSitesResponse struct {
 	Entries []ClusterManagerSitesEntry `json:"entry"`
 }
 
-// ClusterManagerSitesEntry represents a site of an indexer cluster with its metadata
+// ClusterManagerSitesEntry represents a site of an Indexer Cluster with its metadata
 type ClusterManagerSitesEntry struct {
 	Name    string                     `json:"name"`
 	Content ClusterManagerSitesContent `json:"content"`
@@ -55,7 +55,7 @@ type ClusterManagerHealthResponse struct {
 	Entries []ClusterManagerHealthEntry `json:"entry"`
 }
 
-// ClusterManagerHealthEntry represents a site of an indexer cluster with its metadata
+// ClusterManagerHealthEntry represents a site of an Indexer Cluster with its metadata
 type ClusterManagerHealthEntry struct {
 	Name    string                      `json:"name"`
 	Content ClusterManagerHealthContent `json:"content"`
@@ -129,7 +129,7 @@ type ClusterManagerPeersAndSearchHeadResponse struct {
 	} `json:"entry"`
 }
 
-// GetIndexersOrSearchHeadsOnCM get indexers or search head on Cluster Manager
+// GetIndexersOrSearchHeadsOnCM gets Indexers or Search Heads on Cluster Manager
 func GetIndexersOrSearchHeadsOnCM(ctx context.Context, deployment *Deployment, endpoint string) ClusterManagerPeersAndSearchHeadResponse {
 	url := ""
 	if endpoint == "sh" {
@@ -154,7 +154,7 @@ func GetIndexersOrSearchHeadsOnCM(ctx context.Context, deployment *Deployment, e
 	return restResponse
 }
 
-// CheckIndexerOnCM check given Indexer on cluster manager
+// CheckIndexerOnCM checks given Indexer on Cluster Manager
 func CheckIndexerOnCM(ctx context.Context, deployment *Deployment, indexerName string) bool {
 	restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "peer")
 	found := false
@@ -168,7 +168,7 @@ func CheckIndexerOnCM(ctx context.Context, deployment *Deployment, indexerName s
 	return found
 }
 
-// CheckSearchHeadOnCM check given search head on cluster manager
+// CheckSearchHeadOnCM checks given Search Head on Cluster Manager
 func CheckSearchHeadOnCM(ctx context.Context, deployment *Deployment, searchHeadName string) bool {
 	restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh")
 	found := false
@@ -182,7 +182,7 @@ func CheckSearchHeadOnCM(ctx context.Context, deployment *Deployment, searchHead
 	return found
 }
 
-// CheckSearchHeadRemoved check if search head is removed from Indexer Cluster
+// CheckSearchHeadRemoved checks if Search Head is removed from Indexer Cluster
 func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool {
 	restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh")
 	searchHeadRemoved := true

test/testenv/common_test_patterns.go

@@ -162,6 +162,8 @@ type ClusterReadinessConfig struct {
 // function that returns the appropriate ClusterReadinessConfig.
 // This is the standard config type shared by test packages that loop over
 // V3 (master) and V4 (manager) variants.
+// See also MasterManagerLMTestConfig (in lmutil.go) for the license-manager
+// equivalent that returns *LicenseTestConfig instead.
 type MasterManagerTestConfig struct {
 	NamePrefix string
 	Label      string