diff --git a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml index df24959a7..0830d9596 100644 --- a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml +++ b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml @@ -135,7 +135,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-AL2023-int-test-workflow.yml b/.github/workflows/arm-AL2023-int-test-workflow.yml index 39986ebca..a65cf941f 100644 --- a/.github/workflows/arm-AL2023-int-test-workflow.yml +++ b/.github/workflows/arm-AL2023-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-al2023 diff --git a/.github/workflows/arm-RHEL-build-test-push-workflow.yml b/.github/workflows/arm-RHEL-build-test-push-workflow.yml index 07848683c..f86a5e632 100644 --- a/.github/workflows/arm-RHEL-build-test-push-workflow.yml +++ b/.github/workflows/arm-RHEL-build-test-push-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-RHEL-int-test-workflow.yml b/.github/workflows/arm-RHEL-int-test-workflow.yml index 83c7e8fb9..d2a127ca8 100644 --- a/.github/workflows/arm-RHEL-int-test-workflow.yml +++ b/.github/workflows/arm-RHEL-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml index 9d1be84a5..c00f8e73a 100644 --- a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml +++ b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml @@ -135,7 +135,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-Ubuntu-int-test-workflow.yml b/.github/workflows/arm-Ubuntu-int-test-workflow.yml index b3f133abe..8d138f5bb 100644 --- a/.github/workflows/arm-Ubuntu-int-test-workflow.yml +++ b/.github/workflows/arm-Ubuntu-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-ubuntu diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index c398af65c..299a61a9c 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -218,7 +218,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-build-test-push-workflow.yml b/.github/workflows/distroless-build-test-push-workflow.yml index 5da0848a0..b45264645 100644 --- a/.github/workflows/distroless-build-test-push-workflow.yml +++ b/.github/workflows/distroless-build-test-push-workflow.yml @@ -194,7 +194,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-int-test-workflow.yml b/.github/workflows/distroless-int-test-workflow.yml index 01a024247..5f16a8472 100644 --- a/.github/workflows/distroless-int-test-workflow.yml +++ b/.github/workflows/distroless-int-test-workflow.yml @@ -73,7 +73,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-distroless diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 8ffc0ec0c..1e7d04f03 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -70,7 +70,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/.github/workflows/manual-int-test-workflow.yml b/.github/workflows/manual-int-test-workflow.yml index 96629316e..0e3e5d357 100644 --- a/.github/workflows/manual-int-test-workflow.yml +++ b/.github/workflows/manual-int-test-workflow.yml @@ -29,7 +29,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/namespace-scope-int-workflow.yml b/.github/workflows/namespace-scope-int-workflow.yml index fc3d3554f..640f74dc6 100644 --- a/.github/workflows/namespace-scope-int-workflow.yml +++ b/.github/workflows/namespace-scope-int-workflow.yml @@ -25,7 +25,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml index 3f886dd9a..184927f04 100644 --- a/.github/workflows/nightly-int-test-workflow.yml +++ b/.github/workflows/nightly-int-test-workflow.yml @@ -66,7 +66,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/test/appframework_aws/c3/appframework_aws_suite_test.go b/test/appframework_aws/c3/appframework_aws_suite_test.go index dbd361bdd..bd97301e5 100644 --- a/test/appframework_aws/c3/appframework_aws_suite_test.go +++ b/test/appframework_aws/c3/appframework_aws_suite_test.go @@ -17,24 +17,13 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/joho/godotenv" "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -42,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -55,9 +41,7 @@ var ( func TestBasic(t *testing.T) { RegisterFailHandler(Fail) - // Find and load the .env file from the current directory upwards - err := loadEnvFile() - Expect(err).ToNot(HaveOccurred(), "Error loading .env file") + Expect(testenv.LoadEnvFile()).ToNot(HaveOccurred(), "Error loading .env file") sc, _ := GinkgoConfiguration() sc.Timeout = testenv.LongSuiteTimeout @@ -65,77 +49,12 @@ func TestBasic(t *testing.T) { RunSpecs(t, "Running "+testSuiteName, sc) } -//func TestMain(m *testing.M) { -// Run the tests -// os.Exit(m.Run()) -//} - -// loadEnvFile traverses up the directory tree to find a .env file -func loadEnvFile() error { - // Get the current working directory - dir, err := os.Getwd() - if err != nil { - return err - } - - // Traverse up the directory tree - for { - // Check if .env file exists in the current directory - envFile := filepath.Join(dir, ".env") - if _, err := os.Stat(envFile); err == nil { - // .env file found, load it - return godotenv.Load(envFile) - } - - // Move up to the parent directory - parentDir := filepath.Dir(dir) - if parentDir == dir { - // Reached the root directory - return nil - } - dir = parentDir - } -} - var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework_aws/c3/appframework_aws_test.go index 82e078c4b..0ebda2e84 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework_aws/c3/appframework_aws_test.go @@ -24,7 +24,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -52,42 +51,12 @@ var _ = Describe("c3appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 4000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master", testenv.WithTimeout(4000)) + Expect(err).ToNot(HaveOccurred()) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -135,7 +104,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -155,7 +124,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -171,13 +140,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -190,26 +159,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) + Expect(testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment)).To(Succeed(), "Disconnected SH found on CM") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -221,7 +180,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //######### INITIAL VERIFICATIONS ############# var idxcPodNames, shcPodNames []string @@ -234,15 +193,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // get revision number of the resource @@ -269,24 +229,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -302,10 +253,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -352,10 +304,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -376,7 +328,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -392,13 +344,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -411,23 +363,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -443,15 +385,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############## DOWNGRADE APPS ############### // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // get revision number of the resource @@ -478,24 +421,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -511,10 +445,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -574,13 +509,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -590,17 +525,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -629,7 +558,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -639,7 +569,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -647,9 +577,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -662,13 +590,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -679,10 +606,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") // Ensure Indexer Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify New Indexer On Cluster Master indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) @@ -690,23 +617,18 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -724,7 +646,8 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -732,19 +655,17 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -757,11 +678,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -772,21 +692,21 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -803,10 +723,11 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -855,13 +776,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -871,17 +792,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -895,15 +810,16 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 @@ -918,19 +834,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -943,10 +853,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1018,7 +929,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1028,9 +939,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1041,8 +952,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1060,17 +971,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1086,15 +991,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1122,19 +1028,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1153,10 +1053,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1227,7 +1128,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1237,8 +1138,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1248,8 +1149,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1267,17 +1168,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1302,7 +1197,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1312,12 +1208,12 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1345,19 +1241,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1376,15 +1266,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { /* Test Steps @@ -1410,7 +1301,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1429,13 +1320,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1445,17 +1336,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1469,10 +1354,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1523,7 +1409,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1543,7 +1429,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 for Indexer Cluster s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) @@ -1557,35 +1443,29 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1601,15 +1481,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for C3 @@ -1632,24 +1513,18 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1662,10 +1537,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1677,10 +1552,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1692,7 +1567,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1714,10 +1589,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1769,13 +1645,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1784,17 +1660,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1808,15 +1678,16 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 @@ -1831,22 +1702,17 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1860,10 +1726,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1875,10 +1741,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1897,10 +1763,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1970,7 +1837,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1980,8 +1847,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1991,8 +1858,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2009,17 +1876,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2035,15 +1896,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -2083,19 +1945,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2121,10 +1977,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2159,7 +2016,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2179,12 +2036,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2203,13 +2060,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2218,7 +2075,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Cluster Master appList = testenv.ExtraApps @@ -2235,7 +2092,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -2244,16 +2101,16 @@ var _ = Describe("c3appfw test", func() { appList = append(testenv.BigSingleApp, testenv.ExtraApps...) cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify all apps are installed on Deployer appList = append(testenv.BigSingleApp, testenv.ExtraApps...) deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) @@ -2288,7 +2145,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2308,12 +2165,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2332,13 +2189,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2347,7 +2204,7 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Cluster Master appList = testenv.ExtraApps @@ -2364,23 +2221,19 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -2388,7 +2241,7 @@ var _ = Describe("c3appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -2416,7 +2269,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2434,13 +2287,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2450,22 +2303,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2479,10 +2326,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2510,7 +2358,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2528,13 +2376,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2544,22 +2392,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2573,10 +2415,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2619,13 +2462,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2634,17 +2477,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2657,25 +2494,26 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2686,7 +2524,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2723,7 +2561,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2740,13 +2578,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2756,7 +2594,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" @@ -2778,23 +2616,17 @@ var _ = Describe("c3appfw test", func() { //######### VERIFICATIONS ############# appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2806,7 +2638,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2831,7 +2664,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2840,7 +2673,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2851,7 +2684,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2860,14 +2693,14 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -2878,17 +2711,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2902,7 +2729,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2929,7 +2757,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2947,13 +2775,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2963,25 +2791,19 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is completed on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2995,10 +2817,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -3033,13 +2856,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3049,23 +2872,19 @@ var _ = Describe("c3appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) @@ -3106,16 +2925,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Ensure that the Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3126,7 +2945,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -3153,14 +2972,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3179,10 +2998,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -3190,10 +3009,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) // Deploy C3 SVA // Deploy the Cluster Master @@ -3236,17 +3055,11 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3256,19 +3069,21 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // //############### UPGRADE APPS ################ // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster @@ -3278,7 +3093,7 @@ var _ = Describe("c3appfw test", func() { // uploadedApps = append(uploadedApps, uploadedFiles...) // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) + // testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) // // Ensure that the Cluster Master goes to Ready phase // testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework_aws/c3/manager_appframework_test.go index 3c848c41a..34c6317da 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework_aws/c3/manager_appframework_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -48,44 +47,13 @@ var _ = Describe("c3appfw test", func() { var filePresentOnOperator bool BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -133,7 +101,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -153,7 +121,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -169,13 +137,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -188,26 +156,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) + Expect(testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment)).To(Succeed(), "Disconnected SH found on CM") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -219,7 +177,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //######### INITIAL VERIFICATIONS ############# var idxcPodNames, shcPodNames []string @@ -232,15 +190,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // get revision number of the resource @@ -267,24 +226,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -300,10 +250,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -320,15 +271,15 @@ var _ = Describe("c3appfw test", func() { licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() Expect(err).To(Succeed(), "Unable to download license file from S3") // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) + Expect(testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath)).To(Succeed(), "Failed to create license config map") case "azure": licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) Expect(err).To(Succeed(), "Unable to download license file from Azure") // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) + Expect(testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath)).To(Succeed(), "Failed to create license config map") default: fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with provider set as %v", testenv.ClusterProvider)) } // Upload V1 apps to S3 for Monitoring Console @@ -360,22 +311,22 @@ var _ = Describe("c3appfw test", func() { idxc, err := deployment.DeployIndexerCluster(ctx, idxName, lm.GetName(), 3, cm.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") // Wait for License Manager to be in READY phase - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment)).To(Succeed(), "License Manager not ready") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) + Expect(testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment)).To(Succeed(), "Disconnected SH found on CM") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -387,10 +338,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE IMAGE ################ @@ -435,19 +386,19 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for ClusterManager to reach Ready phase") // Ensure Cluster Manager stays in Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for License Manager to be in READY phase - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment)).To(Succeed(), "License Manager not ready") // // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") }) }) @@ -493,10 +444,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -517,7 +468,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -533,13 +484,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -552,23 +503,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -584,15 +525,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############## DOWNGRADE APPS ############### // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // get revision number of the resource @@ -619,24 +561,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -652,10 +585,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -715,13 +649,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -731,17 +665,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -770,7 +698,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -780,7 +709,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -788,9 +717,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -803,13 +730,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -820,10 +746,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") // Ensure Indexer Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Wait for ScaledUp event to confirm scaling completed successfully err = testcaseEnvInst.WaitForScaledUp(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) @@ -835,23 +761,18 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -869,7 +790,8 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -877,19 +799,17 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -902,11 +822,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -917,25 +836,25 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Wait for ScaledDown event to confirm scaling completed successfully err = testcaseEnvInst.WaitForScaledDown(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for ScaledDown event on IndexerCluster") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -952,10 +871,11 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1004,13 +924,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -1020,17 +940,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1044,15 +958,16 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 @@ -1067,19 +982,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1092,10 +1001,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1167,7 +1077,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1177,9 +1087,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1190,8 +1100,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1209,17 +1119,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1235,15 +1139,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1271,19 +1176,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1302,10 +1201,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1376,7 +1276,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1386,8 +1286,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1397,8 +1297,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1416,17 +1316,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1451,7 +1345,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1461,12 +1356,12 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1494,19 +1389,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1525,10 +1414,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1559,7 +1449,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1578,13 +1468,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1594,17 +1484,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1618,10 +1502,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1672,7 +1557,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1692,7 +1577,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 for Indexer Cluster s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) @@ -1706,35 +1591,29 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1750,15 +1629,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for C3 @@ -1781,24 +1661,18 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1811,10 +1685,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1826,10 +1700,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1841,7 +1715,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1863,10 +1737,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1918,13 +1793,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1933,17 +1808,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1957,15 +1826,16 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 @@ -1980,22 +1850,17 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -2009,10 +1874,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -2024,10 +1889,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2046,10 +1911,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2119,7 +1985,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2129,8 +1995,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -2140,8 +2006,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2158,17 +2024,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2184,15 +2044,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -2232,19 +2093,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2270,10 +2125,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2308,7 +2164,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2328,12 +2184,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2352,13 +2208,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2367,7 +2223,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Cluster Manager appList = testenv.ExtraApps @@ -2384,7 +2240,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -2393,16 +2249,16 @@ var _ = Describe("c3appfw test", func() { appList = append(testenv.BigSingleApp, testenv.ExtraApps...) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify all apps are installed on Deployer appList = append(testenv.BigSingleApp, testenv.ExtraApps...) deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) // Vivek need testing @@ -2437,7 +2293,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2457,12 +2313,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2481,13 +2337,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2496,7 +2352,7 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Cluster Manager appList = testenv.ExtraApps @@ -2513,23 +2369,23 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -2537,7 +2393,7 @@ var _ = Describe("c3appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -2565,7 +2421,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2583,13 +2439,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2599,22 +2455,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2628,10 +2478,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2659,7 +2510,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2677,13 +2528,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2693,22 +2544,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2722,10 +2567,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2768,13 +2614,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2783,17 +2629,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2806,25 +2646,26 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2835,7 +2676,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2872,7 +2713,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2889,13 +2730,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2905,7 +2746,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" @@ -2927,23 +2768,17 @@ var _ = Describe("c3appfw test", func() { //######### VERIFICATIONS ############# appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2955,7 +2790,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2980,7 +2816,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2989,7 +2825,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -3000,7 +2836,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -3009,14 +2845,14 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3027,17 +2863,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3051,7 +2881,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3078,7 +2909,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -3096,13 +2927,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3112,25 +2943,19 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3144,10 +2969,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -3182,13 +3008,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3198,23 +3024,19 @@ var _ = Describe("c3appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) @@ -3255,16 +3077,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Ensure that the Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3275,7 +3097,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -3302,14 +3124,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3328,10 +3150,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -3339,10 +3161,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) // Deploy C3 SVA // Deploy the Cluster Manager @@ -3385,17 +3207,11 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3405,19 +3221,21 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // //############### UPGRADE APPS ################ // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster @@ -3427,7 +3245,7 @@ var _ = Describe("c3appfw test", func() { // uploadedApps = append(uploadedApps, uploadedFiles...) // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) + // testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) // // Ensure that the Cluster Manager goes to Ready phase // testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_aws/m4/appframework_aws_suite_test.go b/test/appframework_aws/m4/appframework_aws_suite_test.go index 238d41e4d..44862712f 100644 --- a/test/appframework_aws/m4/appframework_aws_suite_test.go +++ b/test/appframework_aws/m4/appframework_aws_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,54 +39,17 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - sc, _ := GinkgoConfiguration() - sc.Timeout = testenv.LongSuiteTimeout - - RunSpecs(t, "Running "+testSuiteName, sc) + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_aws/m4/appframework_aws_test.go b/test/appframework_aws/m4/appframework_aws_test.go index 3472394e8..55d7c722e 100644 --- a/test/appframework_aws/m4/appframework_aws_test.go +++ b/test/appframework_aws/m4/appframework_aws_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -51,15 +50,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).ToNot(HaveOccurred()) s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -68,30 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { /* Test Steps @@ -132,10 +104,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -154,7 +126,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -168,11 +140,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -184,28 +156,19 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -217,7 +180,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //########## INITIAL VERIFICATIONS ########## var idxcPodNames, shcPodNames []string @@ -230,15 +193,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // get revision number of the resource @@ -265,28 +229,19 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -302,10 +257,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + ClusterMasterBundleHash, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -351,10 +307,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -373,7 +329,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -387,11 +343,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -400,25 +356,16 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -434,16 +381,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete V2 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V1 apps to S3 for Indexer Cluster @@ -467,25 +415,16 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -501,10 +440,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + ClusterMasterBundleHash, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -564,11 +504,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -579,20 +519,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -601,8 +532,8 @@ var _ = Describe("m4appfw test", func() { for i := 1; i <= siteCount; i++ { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") } // ############ Verify livenessProbe and readinessProbe config object and scripts############ @@ -620,10 +551,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -631,9 +563,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -646,13 +576,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -663,28 +592,28 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ingest data on new Indexers podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -702,7 +631,8 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -710,20 +640,18 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -736,11 +664,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -750,21 +677,18 @@ var _ = Describe("m4appfw test", func() { err = deployment.UpdateCR(ctx, idxc) Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -781,10 +705,11 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -830,29 +755,23 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 - indexersPerSite := 1 shReplicas := 3 + indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -866,15 +785,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster @@ -892,16 +812,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -914,10 +828,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -965,10 +880,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -987,7 +902,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -1001,36 +916,27 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1046,12 +952,18 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") + + //############### UPGRADE APPS ################ + // Delete V2 apps on S3 + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - // ############### UPGRADE APPS ################ + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") + uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" @@ -1074,30 +986,21 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1110,13 +1013,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1128,10 +1031,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1143,11 +1046,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) + err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 10*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1160,7 +1063,7 @@ var _ = Describe("m4appfw test", func() { Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - // ############ VERIFY APPS UPDATED TO V2 ############# + //############# UPGRADE APPS ################ appVersion = "V2" cmAppSourceInfo.CrAppVersion = appVersion cmAppSourceInfo.CrAppList = appListV2 @@ -1172,10 +1075,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + ClusterMasterBundleHash, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1225,11 +1129,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -1239,14 +1143,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1260,15 +1158,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster @@ -1286,20 +1185,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1313,13 +1207,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1331,10 +1225,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1355,10 +1249,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1428,7 +1323,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1438,9 +1333,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1450,8 +1345,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1470,20 +1365,11 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1499,15 +1385,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1548,22 +1435,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ########## Verify Manual Poll config map disabled after the poll is triggered ################# @@ -1586,10 +1464,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1623,7 +1502,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1643,12 +1522,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1667,23 +1546,23 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Cluster Master appList = testenv.ExtraApps @@ -1700,7 +1579,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -1710,10 +1589,10 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for SearchHeadCluster to reach Ready phase err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) @@ -1725,7 +1604,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Deployer deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) @@ -1760,7 +1639,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1780,12 +1659,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1804,13 +1683,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1818,10 +1697,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Cluster Master appList = testenv.ExtraApps @@ -1838,23 +1717,23 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -1862,7 +1741,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -1889,7 +1768,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1905,11 +1784,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1917,28 +1796,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1952,10 +1822,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1981,7 +1852,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1997,11 +1868,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2009,28 +1880,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2044,10 +1906,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2087,11 +1950,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2099,22 +1962,13 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2127,28 +1981,29 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2159,7 +2014,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2205,11 +2060,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -2219,14 +2074,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2240,10 +2089,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") @@ -2253,8 +2103,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + Expect(deployment.GetInstance(ctx, deployment.GetName(), cm)).To(Succeed(), "Failed to get instance of Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2264,8 +2113,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2300,7 +2148,7 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster @@ -2323,7 +2171,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2341,10 +2190,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2380,7 +2230,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2395,11 +2245,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2407,10 +2257,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" @@ -2432,26 +2282,26 @@ var _ = Describe("m4appfw test", func() { //########## VERIFICATIONS ########## appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2463,7 +2313,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2487,7 +2338,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2496,7 +2347,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2507,7 +2358,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2516,14 +2367,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -2534,14 +2385,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2555,7 +2400,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2581,7 +2427,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2597,11 +2443,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2609,31 +2455,22 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2647,10 +2484,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2681,11 +2519,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2693,30 +2531,23 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) }) diff --git a/test/appframework_aws/m4/manager_appframework_test.go b/test/appframework_aws/m4/manager_appframework_test.go index 3be4dc9f6..a9deaa7f9 100644 --- a/test/appframework_aws/m4/manager_appframework_test.go +++ b/test/appframework_aws/m4/manager_appframework_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -50,15 +49,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -67,30 +59,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { /* Test Steps @@ -131,10 +103,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -153,7 +125,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -167,11 +139,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -183,28 +155,19 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -216,7 +179,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //########## INITIAL VERIFICATIONS ########## var idxcPodNames, shcPodNames []string @@ -229,15 +192,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // get revision number of the resource @@ -264,28 +228,16 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Verify Monitoring Console is ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -301,10 +253,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -350,10 +303,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -372,7 +325,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -386,11 +339,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -399,25 +352,16 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -433,16 +377,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete V2 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V1 apps to S3 for Indexer Cluster @@ -466,25 +411,16 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -500,10 +436,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -563,11 +500,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -578,20 +515,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -600,8 +528,8 @@ var _ = Describe("m4appfw test", func() { for i := 1; i <= siteCount; i++ { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") } // ############ Verify livenessProbe and readinessProbe config object and scripts############ @@ -619,10 +547,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -630,9 +559,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -645,13 +572,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -662,28 +588,28 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ingest data on new Indexers podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -701,7 +627,8 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -709,20 +636,18 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -735,11 +660,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -749,21 +673,18 @@ var _ = Describe("m4appfw test", func() { err = deployment.UpdateCR(ctx, idxc) Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -780,10 +701,11 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -829,11 +751,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -844,14 +766,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -865,15 +781,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster @@ -891,16 +808,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -913,10 +824,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -964,10 +876,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -986,7 +898,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) @@ -1000,36 +912,28 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1045,10 +949,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ @@ -1073,30 +978,21 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1105,32 +1001,18 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Modify config map to trigger manual update") config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) + config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) err = deployment.UpdateCR(ctx, config) Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1142,10 +1024,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) + err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 10*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1158,7 +1040,7 @@ var _ = Describe("m4appfw test", func() { Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - // ############ VERIFY APPS UPDATED TO V2 ############# + //############### UPGRADE APPS ################ appVersion = "V2" cmAppSourceInfo.CrAppVersion = appVersion cmAppSourceInfo.CrAppList = appListV2 @@ -1170,10 +1052,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1223,11 +1106,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1237,14 +1120,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1258,15 +1135,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster @@ -1284,20 +1162,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1311,13 +1184,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1329,15 +1202,15 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# + // ########## Verify Manual Poll disabled after the poll is triggered ################# // Verify config map set back to off after poll trigger testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) @@ -1353,10 +1226,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1426,7 +1300,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1436,9 +1310,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1448,8 +1322,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1468,20 +1342,11 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1497,15 +1362,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1546,22 +1412,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ########## Verify Manual Poll config map disabled after the poll is triggered ################# @@ -1584,10 +1441,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1621,7 +1479,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1646,7 +1504,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Manager @@ -1670,8 +1528,8 @@ var _ = Describe("m4appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1681,7 +1539,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Cluster Manager appList = testenv.ExtraApps @@ -1758,7 +1616,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1783,7 +1641,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Manager @@ -1807,8 +1665,8 @@ var _ = Describe("m4appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1819,7 +1677,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Cluster Manager appList = testenv.ExtraApps @@ -1887,7 +1745,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1903,11 +1761,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1915,28 +1773,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1950,15 +1799,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { + It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP ################## @@ -1979,7 +1829,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1995,11 +1845,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2007,28 +1857,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2042,10 +1883,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2085,11 +1927,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2097,22 +1939,13 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2125,28 +1958,29 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2157,7 +1991,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2203,11 +2037,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2217,14 +2051,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2238,10 +2066,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster") @@ -2251,8 +2080,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Manager CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") + Expect(deployment.GetInstance(ctx, deployment.GetName(), cm)).To(Succeed(), "Failed to get instance of Cluster Manager") // Set AppsRepoPollInterval for Cluster Manager to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") @@ -2262,8 +2090,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2298,7 +2125,7 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster @@ -2321,7 +2148,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2339,10 +2167,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2378,7 +2207,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2393,11 +2222,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2405,10 +2234,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" @@ -2430,26 +2259,26 @@ var _ = Describe("m4appfw test", func() { //########## VERIFICATIONS ########## appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2461,7 +2290,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2485,7 +2315,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2494,7 +2324,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2505,7 +2335,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2514,14 +2344,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2533,13 +2363,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2553,7 +2383,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2579,7 +2410,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2595,11 +2426,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2607,31 +2438,22 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2645,10 +2467,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2679,11 +2502,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2691,30 +2514,23 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) }) diff --git a/test/appframework_aws/s1/appframework_aws_suite_test.go b/test/appframework_aws/s1/appframework_aws_suite_test.go index 0fe83235d..19ab5b4f3 100644 --- a/test/appframework_aws/s1/appframework_aws_suite_test.go +++ b/test/appframework_aws/s1/appframework_aws_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -63,39 +49,10 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_aws/s1/appframework_aws_test.go b/test/appframework_aws/s1/appframework_aws_test.go index 0a862eb55..43a7a8667 100644 --- a/test/appframework_aws/s1/appframework_aws_test.go +++ b/test/appframework_aws/s1/appframework_aws_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -45,42 +44,15 @@ var _ = Describe("s1appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -128,10 +100,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -150,7 +122,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to S3 for Standalone @@ -162,9 +134,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -183,13 +155,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -201,7 +173,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // ############ INITIAL VERIFICATION ########### standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} @@ -209,13 +181,14 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Standalone and Monitoring Console @@ -232,13 +205,13 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -251,7 +224,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -308,10 +282,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -330,11 +304,11 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -352,13 +326,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -369,12 +343,13 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // get revision number of the resource @@ -394,16 +369,16 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -416,7 +391,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -479,10 +455,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -509,9 +485,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -529,13 +505,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -553,7 +529,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -564,8 +541,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -573,16 +549,17 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady)).To(Succeed(), "Standalone phase mismatch") // Verify Monitoring Console is Ready and stays in ready state //testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) //########### SCALING UP VERIFICATION ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -590,31 +567,31 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //############## SCALING DOWN ################# // Scale down Standalone instance testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) Expect(err).To(Succeed(), "Failed to scale down Standalone") // Ensure Standalone is scaling down - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady)).To(Succeed(), "Standalone phase mismatch") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") //########### SCALING DOWN VERIFICATION ####### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -667,9 +644,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -684,10 +661,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -697,15 +674,15 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### SCALING UP ################## // Scale up Standalone instance testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -713,15 +690,15 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // ############## UPGRADE APPS ################# // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Standalone and Monitoring Console @@ -734,10 +711,10 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -748,7 +725,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -771,7 +749,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to S3 @@ -780,9 +758,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, s3TestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -803,10 +781,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Ensure Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -816,18 +794,19 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Download ES App from S3 testcaseEnvInst.Log.Info("Download updated ES app from S3") - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to S3 for Standalone @@ -838,10 +817,10 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -851,7 +830,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = esApp standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -882,7 +862,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -892,9 +872,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -912,7 +892,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -921,7 +901,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -972,10 +953,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -994,16 +975,16 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to S3 uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 0) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 0) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1021,13 +1002,13 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1036,12 +1017,13 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ //Delete apps on S3 for new Apps - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil //Upload new Versioned Apps to S3 @@ -1059,23 +1041,24 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Step: Wait for Phase Change") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) testcaseEnvInst.Log.Info("Step: Standalone Ready") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") testcaseEnvInst.Log.Info("Step: Verify if Monitoring Console is ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") testcaseEnvInst.Log.Info("Step: Check if verification apps are not updated before enabling manual poll") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1095,10 +1078,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") // Verify Standalone stays in ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1113,7 +1096,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1146,7 +1130,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download the extra apps from S3 for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 for first Standalone @@ -1164,9 +1148,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1178,10 +1162,10 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, s3TestDirStandalone2, 60) + appFrameworkSpecStandalone2 := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, s3TestDirStandalone2, 60) specStandalone2 := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1203,8 +1187,8 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone2) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone2)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1216,12 +1200,13 @@ var _ = Describe("s1appfw test", func() { standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, appframeworksS1, appframework: can add new apps to app source while install is in progress and have all apps installed", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1, appframeworksS1, appframework: can add new apps to app source while install is in progress and have all apps installed", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1248,10 +1233,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1270,13 +1255,13 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1287,9 +1272,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1307,10 +1292,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to S3 for Standalone appList = testenv.ExtraApps @@ -1321,7 +1306,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) @@ -1330,12 +1315,12 @@ var _ = Describe("s1appfw test", func() { appList = append(testenv.BigSingleApp, testenv.ExtraApps...) standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, appframeworksS1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1, appframeworksS1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1355,7 +1340,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1364,9 +1349,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1381,7 +1366,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1393,17 +1378,17 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1412,20 +1397,24 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) + err = wait.PollUntilContextTimeout(ctx, testenv.PollInterval, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + _, getErr := testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) + return getErr == nil, nil + }) Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, appframeworksS1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1, appframeworksS1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1445,7 +1434,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1454,9 +1443,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1471,16 +1460,16 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1489,7 +1478,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1527,9 +1517,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1545,10 +1535,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1557,22 +1547,23 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDir) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileName) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Wait for App state to update after config file change standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) @@ -1584,7 +1575,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -1607,7 +1598,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to S3 App Source @@ -1638,9 +1629,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1657,7 +1648,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) Expect(err).To(Succeed(), "Unable to obtain secret object") @@ -1665,7 +1656,7 @@ var _ = Describe("s1appfw test", func() { modifiedSecretData := map[string][]byte{"s3_access_key": []byte(testenv.RandomDNSName(5)), "s3_secret_key": []byte(testenv.RandomDNSName(5))} // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1674,10 +1665,11 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1685,7 +1677,7 @@ var _ = Describe("s1appfw test", func() { // ############## UPGRADE APPS ################# // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + Expect(testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)).To(Succeed(), "S3 file deletion failed") uploadedApps = nil // Upload V2 apps to S3 for Standalone @@ -1698,10 +1690,11 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Check no apps are updated as auth key is incorrect - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key to correct one########### // Apply spec with correct credentials @@ -1709,10 +1702,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to update secret Object") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1722,13 +1715,14 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, appframeworksS1, appframework: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1, appframeworksS1, appframework: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1750,7 +1744,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to S3 for Standalone @@ -1759,9 +1753,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 120) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 120) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1776,10 +1770,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to S3 for Standalone appVersion = "V2" @@ -1792,13 +1786,13 @@ var _ = Describe("s1appfw test", func() { //######### VERIFICATIONS ############# appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1808,7 +1802,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1833,7 +1828,7 @@ var _ = Describe("s1appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -1842,7 +1837,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 @@ -1854,9 +1849,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1875,7 +1870,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1884,12 +1879,13 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, appframeworksS1, appframework: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1, appframeworksS1, appframework: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1909,7 +1905,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to S3 for Standalone @@ -1918,9 +1914,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1935,19 +1931,19 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1956,7 +1952,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1989,10 +1986,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -2012,10 +2009,10 @@ var _ = Describe("s1appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, mcName, mc.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, mcName, mc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to S3 for Standalone @@ -2027,9 +2024,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2048,16 +2045,16 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") }) }) diff --git a/test/appframework_az/c3/appframework_azure_suite_test.go b/test/appframework_az/c3/appframework_azure_suite_test.go index 7c83c6a1b..0ae939c2a 100644 --- a/test/appframework_az/c3/appframework_azure_suite_test.go +++ b/test/appframework_az/c3/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurec3appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -53,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -63,42 +48,11 @@ func TestBasic(t *testing.T) { } var _ = BeforeSuite(func() { - ctx := context.TODO() var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go index 25ef8617c..3abf3e0a5 100644 --- a/test/appframework_az/c3/appframework_azure_test.go +++ b/test/appframework_az/c3/appframework_azure_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -47,42 +46,12 @@ var _ = Describe("c3appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).ToNot(HaveOccurred()) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -131,7 +100,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -151,7 +120,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -168,13 +137,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -187,26 +156,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) + Expect(testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment)).To(Succeed(), "Disconnected SH found on CM") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -222,16 +181,17 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // get revision number of the resource @@ -259,24 +219,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -292,10 +243,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -342,10 +294,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -366,7 +318,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -382,13 +334,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -401,23 +353,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -433,16 +375,17 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############## DOWNGRADE APPS ############### // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // get revision number of the resource @@ -469,24 +412,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -502,10 +436,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -565,13 +500,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -581,17 +516,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -614,7 +543,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -624,13 +554,11 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -643,13 +571,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -660,10 +587,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") // Ensure Indexer Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify New Indexer On Cluster Manager indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) @@ -671,23 +598,18 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -705,19 +627,18 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -730,11 +651,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -745,21 +665,21 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -776,10 +696,11 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -828,13 +749,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -844,17 +765,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -868,16 +783,17 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure @@ -892,19 +808,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -917,10 +827,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -947,7 +858,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -974,10 +885,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -985,10 +896,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) // Deploy C3 SVA // Deploy the Cluster Master @@ -1031,17 +942,11 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1051,24 +956,26 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ // Delete ES app on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1079,19 +986,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1101,7 +1002,8 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = esApp shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1173,7 +1075,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1183,9 +1085,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1196,8 +1098,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1215,17 +1117,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1250,7 +1146,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1260,13 +1157,13 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1294,19 +1191,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1325,10 +1216,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1402,7 +1294,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1412,8 +1304,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1423,8 +1315,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1442,17 +1334,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1472,16 +1358,17 @@ var _ = Describe("c3appfw test", func() { err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1509,19 +1396,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1540,10 +1421,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1574,7 +1456,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1594,13 +1476,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1610,17 +1492,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1634,10 +1510,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1688,7 +1565,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1708,7 +1585,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure for Indexer Cluster azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) @@ -1722,35 +1599,29 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1766,16 +1637,17 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for C3 @@ -1798,24 +1670,18 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1828,10 +1694,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1843,10 +1709,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1858,7 +1724,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1880,10 +1746,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1935,13 +1802,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1950,17 +1817,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1974,16 +1835,17 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure @@ -1998,22 +1860,17 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -2027,10 +1884,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -2042,10 +1899,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2064,10 +1921,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2137,7 +1995,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2147,8 +2005,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -2158,8 +2016,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2176,17 +2034,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2202,16 +2054,17 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -2251,19 +2104,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2289,10 +2136,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2327,7 +2175,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2347,12 +2195,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2372,13 +2220,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2387,7 +2235,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Cluster Manager appList = testenv.ExtraApps @@ -2404,7 +2252,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -2413,16 +2261,16 @@ var _ = Describe("c3appfw test", func() { appList = append(testenv.BigSingleApp, testenv.ExtraApps...) cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify all apps are installed on Deployer appList = append(testenv.BigSingleApp, testenv.ExtraApps...) deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) @@ -2457,7 +2305,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2477,12 +2325,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2502,13 +2350,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2517,7 +2365,7 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Cluster Manager appList = testenv.ExtraApps @@ -2534,23 +2382,23 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -2558,7 +2406,7 @@ var _ = Describe("c3appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -2586,7 +2434,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2605,13 +2453,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2621,22 +2469,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2650,10 +2492,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2681,7 +2524,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2700,13 +2543,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2716,22 +2559,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2745,10 +2582,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2792,13 +2630,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2807,17 +2645,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2830,12 +2662,13 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") @@ -2843,13 +2676,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2861,7 +2694,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2898,7 +2731,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2916,13 +2749,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2932,7 +2765,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to Azure for Indexer Cluster appVersion = "V2" @@ -2954,23 +2787,17 @@ var _ = Describe("c3appfw test", func() { //######### VERIFICATIONS ############# appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2982,7 +2809,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3008,7 +2836,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -3023,7 +2851,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) @@ -3032,14 +2860,14 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3050,17 +2878,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3074,7 +2896,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3101,7 +2924,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3120,13 +2943,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3136,25 +2959,19 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3168,10 +2985,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -3206,13 +3024,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3222,23 +3040,19 @@ var _ = Describe("c3appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) @@ -3279,16 +3093,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Ensure that the Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3299,7 +3113,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) }) diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go index 71bb91080..f5c938fdf 100644 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ b/test/appframework_az/c3/manager_appframework_azure_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -46,42 +45,12 @@ var _ = Describe("c3appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -130,7 +99,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -150,7 +119,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -166,13 +135,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -185,26 +154,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) + Expect(testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment)).To(Succeed(), "Disconnected SH found on CM") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -220,16 +179,17 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // get revision number of the resource @@ -257,24 +217,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -290,10 +241,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -340,10 +292,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -364,7 +316,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -380,13 +332,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -399,23 +351,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -431,16 +373,17 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############## DOWNGRADE APPS ############### // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // get revision number of the resource @@ -467,24 +410,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -500,10 +434,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -563,13 +498,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -579,17 +514,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -612,7 +541,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -622,13 +552,11 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -641,13 +569,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -658,10 +585,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") // Ensure Indexer Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify New Indexer On Cluster Manager indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) @@ -669,23 +596,18 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -703,19 +625,18 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -728,11 +649,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -743,21 +663,21 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -774,10 +694,11 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -826,13 +747,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -842,17 +763,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -866,16 +781,17 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure @@ -890,19 +806,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -915,10 +825,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -945,7 +856,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -972,10 +883,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -983,10 +894,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) // Deploy C3 SVA // Deploy the Cluster Master @@ -1029,17 +940,11 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1049,24 +954,26 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ // Delete ES app on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1077,19 +984,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1099,7 +1000,8 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = esApp shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1172,7 +1074,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1182,9 +1084,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1195,8 +1097,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1214,17 +1116,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1249,7 +1145,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1259,13 +1156,13 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1293,19 +1190,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1324,10 +1215,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1401,7 +1293,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1411,8 +1303,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1422,8 +1314,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1441,17 +1333,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1472,16 +1358,17 @@ var _ = Describe("c3appfw test", func() { err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1509,19 +1396,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1540,10 +1421,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1574,7 +1456,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1594,13 +1476,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1610,17 +1492,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1634,10 +1510,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1688,7 +1565,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1708,7 +1585,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure for Indexer Cluster azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) @@ -1722,35 +1599,29 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1766,16 +1637,17 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for C3 @@ -1798,24 +1670,18 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1828,10 +1694,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1843,10 +1709,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1858,7 +1724,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1880,10 +1746,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1935,13 +1802,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1950,17 +1817,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1974,16 +1835,17 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure @@ -1998,22 +1860,17 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -2027,10 +1884,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -2042,10 +1899,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2064,10 +1921,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2137,7 +1995,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2147,8 +2005,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -2158,8 +2016,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2176,17 +2034,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2202,16 +2054,17 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -2251,19 +2104,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2289,10 +2136,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2327,7 +2175,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2347,12 +2195,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2372,13 +2220,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2387,7 +2235,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Cluster Manager appList = testenv.ExtraApps @@ -2404,7 +2252,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -2413,16 +2261,16 @@ var _ = Describe("c3appfw test", func() { appList = append(testenv.BigSingleApp, testenv.ExtraApps...) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify all apps are installed on Deployer appList = append(testenv.BigSingleApp, testenv.ExtraApps...) deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) @@ -2457,7 +2305,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2477,12 +2325,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2502,13 +2350,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2517,7 +2365,7 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Cluster Manager appList = testenv.ExtraApps @@ -2534,23 +2382,23 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -2558,7 +2406,7 @@ var _ = Describe("c3appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -2586,7 +2434,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2605,13 +2453,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2621,22 +2469,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2650,10 +2492,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2681,7 +2524,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2700,13 +2543,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2716,22 +2559,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2745,10 +2582,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2792,13 +2630,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2807,17 +2645,11 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2830,12 +2662,13 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") @@ -2843,13 +2676,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2861,7 +2694,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2898,7 +2731,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2916,13 +2749,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2932,7 +2765,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to Azure for Indexer Cluster appVersion = "V2" @@ -2954,23 +2787,17 @@ var _ = Describe("c3appfw test", func() { //######### VERIFICATIONS ############# appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2982,7 +2809,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3008,7 +2836,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -3023,7 +2851,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) @@ -3032,14 +2860,14 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3050,17 +2878,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3074,7 +2896,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3101,7 +2924,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3120,13 +2943,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3136,25 +2959,19 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3168,10 +2985,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -3206,13 +3024,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3222,23 +3040,19 @@ var _ = Describe("c3appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) @@ -3279,16 +3093,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance with ClusterManagerRef") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Ensure that the Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3299,7 +3113,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) }) diff --git a/test/appframework_az/m4/appframework_azure_suite_test.go b/test/appframework_az/m4/appframework_azure_suite_test.go index b66d4ac7f..302f200a9 100644 --- a/test/appframework_az/m4/appframework_azure_suite_test.go +++ b/test/appframework_az/m4/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurem4appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -53,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -63,46 +48,11 @@ func TestBasic(t *testing.T) { } var _ = BeforeSuite(func() { - ctx := context.TODO() var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_az/m4/appframework_azure_test.go b/test/appframework_az/m4/appframework_azure_test.go index 1cf9e5de8..d09246e2f 100644 --- a/test/appframework_az/m4/appframework_azure_test.go +++ b/test/appframework_az/m4/appframework_azure_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -50,15 +49,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).ToNot(HaveOccurred()) azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -67,31 +59,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { /* Test Steps @@ -132,10 +103,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -154,7 +125,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -168,11 +139,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -184,28 +155,15 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -221,16 +179,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Indexer Cluster @@ -254,28 +213,19 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -291,10 +241,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -340,10 +291,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -362,7 +313,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -376,11 +327,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -389,25 +340,16 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -423,17 +365,18 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete V2 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V1 apps to Azure for Indexer Cluster @@ -457,25 +400,16 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -491,10 +425,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -554,11 +489,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -569,20 +504,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -591,8 +517,8 @@ var _ = Describe("m4appfw test", func() { for i := 1; i <= siteCount; i++ { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") } //########### INITIAL VERIFICATIONS ######### @@ -604,16 +530,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -626,13 +551,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -643,28 +567,28 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ingest data on new Indexers podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -682,20 +606,19 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -708,11 +631,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -723,20 +645,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -753,10 +675,11 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -802,11 +725,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -817,14 +740,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -838,16 +755,17 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Indexer Cluster @@ -865,16 +783,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -887,10 +799,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -938,10 +851,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -960,7 +873,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -974,36 +887,27 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1019,10 +923,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ @@ -1047,30 +952,21 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1083,13 +979,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1101,10 +997,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1116,10 +1012,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) + err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 10*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1144,10 +1040,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1197,11 +1094,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1211,14 +1108,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1232,16 +1123,17 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Indexer Cluster @@ -1259,20 +1151,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1286,13 +1173,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1304,10 +1191,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1328,10 +1215,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1401,7 +1289,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1411,9 +1299,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1423,8 +1311,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1443,20 +1331,11 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1472,16 +1351,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1522,22 +1402,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ########## Verify Manual Poll config map disabled after the poll is triggered ################# @@ -1560,10 +1431,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1597,7 +1469,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1617,12 +1489,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1642,23 +1514,23 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Cluster Manager appList = testenv.ExtraApps @@ -1675,7 +1547,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -1685,10 +1557,10 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for SearchHeadCluster to reach Ready phase err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) @@ -1699,7 +1571,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Deployer deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) @@ -1734,7 +1606,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1754,12 +1626,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1779,13 +1651,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1793,10 +1665,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Cluster Manager appList = testenv.ExtraApps @@ -1813,23 +1685,23 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -1837,7 +1709,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -1864,7 +1736,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1881,11 +1753,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1893,28 +1765,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1928,10 +1791,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1957,7 +1821,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1974,11 +1838,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1986,28 +1850,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2021,10 +1876,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2065,11 +1921,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2077,22 +1933,13 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2105,12 +1952,13 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") @@ -2118,16 +1966,16 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2139,7 +1987,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2185,11 +2033,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2199,14 +2047,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2220,10 +2062,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") @@ -2233,8 +2076,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + Expect(deployment.GetInstance(ctx, deployment.GetName(), cm)).To(Succeed(), "Failed to get instance of Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2244,8 +2086,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2281,7 +2122,7 @@ var _ = Describe("m4appfw test", func() { // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Indexer Cluster @@ -2304,7 +2145,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2322,10 +2164,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2361,7 +2204,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2377,11 +2220,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2389,10 +2232,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to Azure for Indexer Cluster appVersion = "V2" @@ -2414,26 +2257,17 @@ var _ = Describe("m4appfw test", func() { //########## VERIFICATIONS ########## appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2445,7 +2279,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2470,7 +2305,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2485,7 +2320,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") @@ -2494,14 +2329,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2512,14 +2347,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2533,7 +2362,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2559,7 +2389,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2576,11 +2406,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2588,31 +2418,22 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2626,10 +2447,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2660,11 +2482,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2672,30 +2494,23 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) diff --git a/test/appframework_az/m4/manager_appframework_azure_test.go b/test/appframework_az/m4/manager_appframework_azure_test.go index b4708e6cc..7a958a4d3 100644 --- a/test/appframework_az/m4/manager_appframework_azure_test.go +++ b/test/appframework_az/m4/manager_appframework_azure_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -49,15 +48,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -66,28 +58,7 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { @@ -131,10 +102,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -153,7 +124,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -167,11 +138,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -183,28 +154,15 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -220,16 +178,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Indexer Cluster @@ -253,28 +212,19 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -290,10 +240,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -339,10 +290,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -361,7 +312,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -375,11 +326,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -388,25 +339,16 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -422,17 +364,18 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete V2 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V1 apps to Azure for Indexer Cluster @@ -456,25 +399,16 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -490,10 +424,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -553,11 +488,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -568,20 +503,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -590,8 +516,8 @@ var _ = Describe("m4appfw test", func() { for i := 1; i <= siteCount; i++ { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") } //########### INITIAL VERIFICATIONS ######### @@ -603,16 +529,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -625,13 +550,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -642,28 +566,28 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ingest data on new Indexers podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -681,20 +605,19 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -707,11 +630,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -722,20 +644,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -752,10 +674,11 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -801,11 +724,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -816,14 +739,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -837,16 +754,17 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Indexer Cluster @@ -864,16 +782,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -886,10 +798,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -937,10 +850,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -959,7 +872,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) @@ -973,36 +886,27 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1018,10 +922,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ @@ -1046,30 +951,21 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1082,13 +978,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1100,10 +996,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1115,10 +1011,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) + err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 10*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1143,10 +1039,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1196,11 +1093,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1210,14 +1107,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1231,16 +1122,17 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Indexer Cluster @@ -1258,20 +1150,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1285,13 +1172,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1303,10 +1190,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1327,10 +1214,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1400,7 +1288,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1410,9 +1298,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1422,8 +1310,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1442,20 +1330,11 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1471,16 +1350,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1521,22 +1401,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ########## Verify Manual Poll config map disabled after the poll is triggered ################# @@ -1559,10 +1430,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1596,7 +1468,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1616,12 +1488,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1641,23 +1513,23 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Cluster Manager appList = testenv.ExtraApps @@ -1674,7 +1546,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -1684,10 +1556,10 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for SearchHeadCluster to reach Ready phase err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) @@ -1699,7 +1571,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Deployer deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) @@ -1734,7 +1606,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1754,12 +1626,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1779,13 +1651,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1793,10 +1665,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Cluster Manager appList = testenv.ExtraApps @@ -1813,23 +1685,23 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -1837,7 +1709,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -1864,7 +1736,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1881,11 +1753,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1893,28 +1765,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1928,10 +1791,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1957,7 +1821,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1974,11 +1838,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1986,28 +1850,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2021,10 +1876,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2065,11 +1921,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2077,22 +1933,13 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2105,12 +1952,13 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") @@ -2118,16 +1966,16 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2139,7 +1987,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2185,11 +2033,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2199,14 +2047,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2220,10 +2062,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") @@ -2233,8 +2076,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + Expect(deployment.GetInstance(ctx, deployment.GetName(), cm)).To(Succeed(), "Failed to get instance of Cluster Manager") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2244,8 +2086,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2281,7 +2122,7 @@ var _ = Describe("m4appfw test", func() { // Delete V1 apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Indexer Cluster @@ -2304,7 +2145,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2322,10 +2164,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2361,7 +2204,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2377,11 +2220,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2389,10 +2232,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to Azure for Indexer Cluster appVersion = "V2" @@ -2414,26 +2257,17 @@ var _ = Describe("m4appfw test", func() { //########## VERIFICATIONS ########## appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2445,7 +2279,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2470,7 +2305,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2485,7 +2320,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") @@ -2494,14 +2329,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2512,14 +2347,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2533,7 +2362,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2559,7 +2389,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2576,11 +2406,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2588,31 +2418,22 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2626,10 +2447,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2660,11 +2482,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2672,30 +2494,23 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) diff --git a/test/appframework_az/s1/appframework_azure_suite_test.go b/test/appframework_az/s1/appframework_azure_suite_test.go index f6f891805..9aae5f9f4 100644 --- a/test/appframework_az/s1/appframework_azure_suite_test.go +++ b/test/appframework_az/s1/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azures1appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -53,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -63,42 +48,11 @@ func TestBasic(t *testing.T) { } var _ = BeforeSuite(func() { - ctx := context.TODO() var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_az/s1/appframework_azure_test.go b/test/appframework_az/s1/appframework_azure_test.go index 4a33d1b61..7aa94c775 100644 --- a/test/appframework_az/s1/appframework_azure_test.go +++ b/test/appframework_az/s1/appframework_azure_test.go @@ -19,7 +19,6 @@ import ( "strings" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -40,44 +39,15 @@ var _ = Describe("s1appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) azTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -128,10 +98,10 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) appFrameworkSpecMC.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -151,7 +121,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to Azure for Standalone @@ -161,9 +131,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -182,13 +152,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -199,14 +169,15 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil appVersion = "V2" @@ -222,13 +193,13 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -241,7 +212,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -297,10 +269,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -319,11 +291,11 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -341,13 +313,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -358,13 +330,14 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############# DOWNGRADE APPS ################ // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // get revision number of the resource @@ -384,16 +357,16 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -406,7 +379,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -471,10 +445,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -493,7 +467,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload apps to Azure for Standalone azTestDir := "azures1appfw-" + testenv.RandomDNSName(4) @@ -502,9 +476,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -522,13 +496,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -540,15 +514,15 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### SCALING UP ################## // Scale up Standalone instance testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -556,40 +530,41 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady)).To(Succeed(), "Standalone phase mismatch") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") //########### SCALING UP VERIFICATION ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############## SCALING DOWN ################# // Scale down Standalone instance testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) Expect(err).To(Succeed(), "Failed to scale down Standalone") // Ensure Standalone is scaling down - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady)).To(Succeed(), "Standalone phase mismatch") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") //########### SCALING DOWN VERIFICATION ####### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -643,9 +618,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -660,10 +635,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -673,15 +648,15 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### SCALING UP ################## // Scale up Standalone instance testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -689,16 +664,16 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // ############## UPGRADE APPS ################# // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Standalone and Monitoring Console @@ -711,10 +686,10 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -725,7 +700,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -748,7 +724,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -759,9 +735,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -782,7 +758,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -792,17 +768,18 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") // Download ES App from Azure - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -814,10 +791,10 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -827,7 +804,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = esApp standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -858,7 +836,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -869,9 +847,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -889,7 +867,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -898,7 +876,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -949,10 +928,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -971,16 +950,16 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to Azure uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -998,13 +977,13 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1013,13 +992,14 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ //Delete apps on Azure for new Apps azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil //Upload new Versioned Apps to Azure @@ -1035,17 +1015,18 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1061,10 +1042,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1079,7 +1060,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1112,7 +1094,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download the extra apps from Azure for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1131,9 +1113,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1145,10 +1127,10 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) + appFrameworkSpecStandalone2 := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) specStandalone2 := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1170,8 +1152,8 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone2) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone2)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1182,12 +1164,13 @@ var _ = Describe("s1appfw test", func() { standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can add new apps to app source while install is in progress and have all apps installed", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1azure, appframeworkazures1, appframework: can add new apps to app source while install is in progress and have all apps installed", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1215,10 +1198,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1237,13 +1220,13 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1255,9 +1238,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1275,10 +1258,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to Azure for Standalone appList = testenv.ExtraApps @@ -1289,7 +1272,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) @@ -1298,12 +1281,12 @@ var _ = Describe("s1appfw test", func() { appList = append(testenv.BigSingleApp, testenv.ExtraApps...) standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1323,7 +1306,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1334,9 +1317,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1351,16 +1334,16 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1369,12 +1352,13 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1394,7 +1378,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1405,9 +1389,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1422,16 +1406,16 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1440,7 +1424,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1479,9 +1464,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1497,10 +1482,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1509,12 +1494,13 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") @@ -1522,10 +1508,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileName) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Wait for App state to update after config file change standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) @@ -1538,7 +1524,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -1561,7 +1547,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to Azure App Source @@ -1592,9 +1578,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1611,14 +1597,14 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, _ := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) secretData := secretStruct.Data modifiedSecretData := map[string][]byte{"azure_sa_name": []byte(testenv.RandomDNSName(5)), "azure_sa_secret_key": []byte(testenv.RandomDNSName(5))} // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1627,10 +1613,11 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1639,7 +1626,7 @@ var _ = Describe("s1appfw test", func() { // Delete apps on Azure testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + Expect(azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps)).To(Succeed(), "Azure file deletion failed") uploadedApps = nil // Upload V2 apps to Azure for Standalone @@ -1652,10 +1639,11 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Check no apps are updated as auth key is incorrect - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key to correct one########### // Apply spec with correct credentials @@ -1663,10 +1651,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to update secret Object") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1676,13 +1664,14 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1704,7 +1693,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1714,9 +1703,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1731,10 +1720,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to Azure for Standalone appVersion = "V2" @@ -1746,27 +1735,28 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) //######### VERIFICATIONS ############# appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") //############ UPGRADE VERIFICATION ########### appVersion = "V2" standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1792,7 +1782,7 @@ var _ = Describe("s1appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -1808,9 +1798,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1829,7 +1819,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1838,12 +1828,13 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1863,7 +1854,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big app") @@ -1873,9 +1864,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1890,19 +1881,19 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1911,7 +1902,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1944,10 +1936,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1967,10 +1959,10 @@ var _ = Describe("s1appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, mcName, mc.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, mcName, mc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to Azure for Standalone @@ -1982,9 +1974,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2003,16 +1995,16 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") }) }) diff --git a/test/appframework_gcp/c3/appframework_gcs_suite_test.go b/test/appframework_gcp/c3/appframework_gcs_suite_test.go index 226ea5d49..5f89b27a0 100644 --- a/test/appframework_gcp/c3/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -63,43 +49,10 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to Gcs - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Gcs after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files.") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files.") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go index a21387526..4beb31bbc 100644 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_test.go @@ -14,10 +14,8 @@ package c3gcpappfw import ( - //"encoding/json" "fmt" - "path/filepath" //"strings" //"time" @@ -26,7 +24,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -52,44 +49,13 @@ var _ = Describe("c3appfw test", func() { var filePresentOnOperator bool BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 5000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master", testenv.WithTimeout(5000)) + Expect(err).ToNot(HaveOccurred()) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -137,7 +103,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -157,7 +123,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to GCS for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Bucket for Indexer Cluster", appVersion)) @@ -173,13 +139,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -192,26 +158,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) + Expect(testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment)).To(Succeed(), "Disconnected SH found on CM") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -223,7 +179,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //######### INITIAL VERIFICATIONS ############# var idxcPodNames, shcPodNames []string @@ -236,15 +192,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on GCS testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // get revision number of the resource @@ -271,24 +228,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -304,10 +252,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -354,10 +303,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -378,7 +327,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to GCS for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) @@ -394,13 +343,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -413,23 +362,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -445,15 +384,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############## DOWNGRADE APPS ############### // Delete apps on GCS testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // get revision number of the resource @@ -480,24 +420,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -513,10 +444,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -564,13 +496,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -580,17 +512,11 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -604,15 +530,16 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on GCS testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to GCS @@ -627,19 +554,13 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -652,10 +573,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -689,13 +611,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -705,23 +627,19 @@ var _ = Describe("c3appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "C3 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) }) diff --git a/test/appframework_gcp/c3/manager_appframework_test.go b/test/appframework_gcp/c3/manager_appframework_test.go index 7ad2ffd13..f3631ef80 100644 --- a/test/appframework_gcp/c3/manager_appframework_test.go +++ b/test/appframework_gcp/c3/manager_appframework_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -48,45 +47,13 @@ var _ = Describe("c3appfw test", func() { var filePresentOnOperator bool BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - testenv.SpecifiedTestTimeout = 100000 + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "", testenv.WithTimeout(100000)) + Expect(err).ToNot(HaveOccurred()) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Gcs - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -94,10 +61,10 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -109,7 +76,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for Monitoring Console and C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -122,19 +89,19 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -154,29 +121,29 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -190,25 +157,25 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) + Expect(testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment)).To(Succeed(), "Disconnected SH found on CM") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -220,7 +187,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //######### INITIAL VERIFICATIONS ############# var idxcPodNames, shcPodNames []string @@ -233,59 +200,60 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Upload V2 apps to Gcs for Indexer Cluster + // Upload V2 apps to GCS for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V2 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -301,10 +269,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -319,25 +288,25 @@ var _ = Describe("c3appfw test", func() { switch testenv.ClusterProvider { case "eks": licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from Gcs") + Expect(err).To(Succeed(), "Unable to download license file from GCS") // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) + Expect(testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath)).To(Succeed(), "Failed to create license config map") case "azure": licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) Expect(err).To(Succeed(), "Unable to download license file from Azure") // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) + Expect(testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath)).To(Succeed(), "Failed to create license config map") case "gcp": licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() Expect(err).To(Succeed(), "Unable to download license file from GCP") // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) + Expect(testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath)).To(Succeed(), "Failed to create license config map") default: fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with provider set as %v", testenv.ClusterProvider)) } - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console oldImage := "Refer to RELATED_SPLUNK_IMAGE_ENTERPRISE" newImage := "splunk/splunk:latest" @@ -366,25 +335,25 @@ var _ = Describe("c3appfw test", func() { idxc, err := deployment.DeployIndexerCluster(ctx, idxName, lm.GetName(), 3, cm.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") // Wait for License Manager to be in READY phase - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment)).To(Succeed(), "License Manager not ready") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Monitoring Console goes to Ready phase - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) + Expect(testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment)).To(Succeed(), "Disconnected SH found on CM") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -396,10 +365,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE IMAGE ################ @@ -439,19 +408,19 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed upgrade Indexer Cluster image") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for License Manager to be in READY phase - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment)).To(Succeed(), "License Manager not ready") // // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") }) }) @@ -461,10 +430,10 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V2 apps to Gcs for Monitoring Console + * Upload V2 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster * Create app source for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ########### INITIAL VERIFICATIONS ########### @@ -476,7 +445,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods ############## DOWNGRADE APPS ############### - * Upload V1 apps on Gcs + * Upload V1 apps on GCS * Wait for Monitoring Console and C3 pods to be ready ########### FINAL VERIFICATIONS ############# * Verify Apps are Downloaded in App Deployment Info @@ -489,19 +458,19 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V2 apps to Gcs for Monitoring Console + // Upload V2 apps to GCS for Monitoring Console appVersion := "V2" appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -522,29 +491,29 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") - // Upload V2 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V2 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -558,22 +527,22 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -589,59 +558,60 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############## DOWNGRADE APPS ############### - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexers", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexers", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V1 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -657,10 +627,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -669,7 +640,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps on Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps on GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app config and wait for pods to be ready ########## INITIAL VERIFICATIONS ############ @@ -704,29 +675,29 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP ################## - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -737,16 +708,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -775,7 +746,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -785,7 +757,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -793,9 +765,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -808,13 +778,12 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -825,10 +794,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") // Ensure Indexer Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify New Indexer On Cluster Manager indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) @@ -836,23 +805,18 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -870,7 +834,8 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -878,19 +843,17 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -903,11 +866,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -918,21 +880,21 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -949,10 +911,11 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -962,7 +925,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############# INITIAL VERIFICATIONS ########## @@ -985,29 +948,29 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -1018,16 +981,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1041,42 +1004,43 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil - // Upload V2 apps to Gcs + // Upload V2 apps to GCS appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) appFileList = testenv.GetAppFileList(appListV2) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1089,10 +1053,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1102,7 +1067,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -1114,7 +1079,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -1132,39 +1097,39 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1174,9 +1139,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1187,8 +1152,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1207,16 +1172,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1232,55 +1197,56 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) + // Upload appListLocal list of V2 apps to GCS (to be used for local install) appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1299,10 +1265,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1313,7 +1280,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -1325,7 +1292,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### Downgrade APPS ################ - * Upload V1 apps on Gcs + * Upload V1 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -1343,37 +1310,37 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV2[len(appListV2)/2:] appListCluster := appListV2[:len(appListV2)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Idxc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Shc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1383,8 +1350,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1394,8 +1361,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1414,16 +1381,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1448,7 +1415,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1458,52 +1426,52 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V1 apps to Gcs (to be used for local install) + // Upload appListLocal list of V1 apps to GCS (to be used for local install) appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1522,10 +1490,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1536,7 +1505,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Gcs for this test + * Add more apps than usual on GCS for this test * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############### VERIFICATIONS ############### * Verify Apps are Downloaded in App Deployment Info @@ -1554,34 +1523,34 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appList) appVersion := "V1" - // Download apps from Gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from Gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + // Download apps from GCS + testcaseEnvInst.Log.Info("Download bigger amount of apps from GCS for this test") + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files appList = append(appListV1, appList...) appFileList = testenv.GetAppFileList(appList) - // Upload app to Gcs for Indexer Cluster + // Upload app to GCS for Indexer Cluster gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Indexer Cluster") + Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload app to Gcs for Search Head Cluster + // Upload app to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1592,16 +1561,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1615,10 +1584,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1626,10 +1596,10 @@ var _ = Describe("c3appfw test", func() { It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled for manual update", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ########## INITIAL VERIFICATION ############# @@ -1657,19 +1627,19 @@ var _ = Describe("c3appfw test", func() { */ // ################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1689,49 +1659,49 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1747,55 +1717,56 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil - // Upload V2 apps to Gcs for C3 + // Upload V2 apps to GCS for C3 appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to GCS", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V2 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1808,10 +1779,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1823,10 +1794,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1838,7 +1809,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1860,10 +1831,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1872,7 +1844,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############# INITIAL VERIFICATION ########## @@ -1899,29 +1871,29 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1931,16 +1903,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1954,45 +1926,47 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil - // Upload V2 apps to Gcs + // Upload V2 apps to GCS appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) appFileList = testenv.GetAppFileList(appListV2) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -2006,10 +1980,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -2021,10 +1995,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2043,10 +2017,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2056,7 +2031,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -2068,7 +2043,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -2086,37 +2061,37 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Idxc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Shc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2126,8 +2101,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -2137,8 +2112,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2156,16 +2131,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2181,39 +2156,40 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) + // Upload appListLocal list of V2 apps to GCS (to be used for local install) appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // ############ ENABLE MANUAL POLL ############ @@ -2229,19 +2205,19 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2267,10 +2243,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2279,33 +2256,33 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework ############## VERIFICATIONS ################ * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install + * Upload more apps from GCS during bigger app install * Wait for polling interval to pass * Verify all apps are installed on Cluster Manager and Deployer */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2325,37 +2302,37 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big-size app to Gcs for Cluster Manager + // Upload big-size app to GCS for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Cluster Manager") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") + // Upload big-size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2364,24 +2341,24 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") - // Upload more apps to Gcs for Cluster Manager + // Upload more apps to GCS for Cluster Manager appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload more apps to GCS for Cluster Manager") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") + // Upload more apps to GCS for Deployer + testcaseEnvInst.Log.Info("Upload more apps to GCS for Deployer") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -2390,16 +2367,16 @@ var _ = Describe("c3appfw test", func() { appList = append(testenv.BigSingleApp, testenv.ExtraApps...) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify all apps are installed on Deployer appList = append(testenv.BigSingleApp, testenv.ExtraApps...) deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) // Vivek need testing @@ -2408,33 +2385,33 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ############## VERIFICATIONS ################ * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install + * Upload more apps from GCS during bigger app install * Wait for polling interval to pass * Verify all apps are installed on Cluster Manager and Deployer */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2454,37 +2431,37 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big-size app to Gcs for Cluster Manager + // Upload big-size app to GCS for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Cluster Manager") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") + // Upload big-size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2493,40 +2470,40 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") - // Upload more apps to Gcs for Cluster Manager + // Upload more apps to GCS for Cluster Manager appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload more apps to GCS for Cluster Manager") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") + // Upload more apps to GCS for Deployer + testcaseEnvInst.Log.Info("Upload more apps to GCS for Deployer") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -2534,7 +2511,7 @@ var _ = Describe("c3appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -2544,7 +2521,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app install is in progress, restart the operator @@ -2559,34 +2536,34 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2596,22 +2573,22 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2625,10 +2602,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2638,7 +2616,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app download is in progress, restart the operator @@ -2653,34 +2631,34 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2690,22 +2668,22 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2719,10 +2697,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2732,7 +2711,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### VERIFICATIONS ############# @@ -2744,34 +2723,34 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods * Disable the app - * Delete the app from Gcs + * Delete the app from GCS * Check for repo state in App Deployment Info */ //################## SETUP #################### appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2781,16 +2760,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2803,36 +2782,37 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") - // Verify repo state on App to be disabled to be 1 (i.e app present on Gcs bucket) + // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on GCS") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - // Delete the file from Gcs + // Delete the file from GCS gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Gcs test directory", appFileName[0])) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCS test directory", appFileName[0])) - // Verify repo state is set to 2 (i.e app deleted from Gcs bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + // Verify repo state is set to 2 (i.e app deleted from GCS bucket) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2842,7 +2822,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app download is completed, upload new versions of the apps @@ -2865,34 +2845,34 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2902,21 +2882,21 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") - // Upload V2 apps to Gcs for Indexer Cluster + // Upload V2 apps to GCS for Indexer Cluster appVersion = "V2" appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Get Pod age to check for pod resets later @@ -2924,23 +2904,23 @@ var _ = Describe("c3appfw test", func() { //######### VERIFICATIONS ############# appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2952,7 +2932,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2962,7 +2943,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload 15 apps of 100MB size each to Gcs for Indexer Cluster and Search Head Cluster for cluster scope + * Upload 15 apps of 100MB size each to GCS for Indexer Cluster and Search Head Cluster for cluster scope * Create app sources for Cluster Master and Deployer with cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -2977,7 +2958,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2986,34 +2967,34 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") - // Upload apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search head Cluster", appVersion)) + // Upload apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3025,16 +3006,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3048,7 +3029,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3057,7 +3039,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * When app download is complete, delete apps from app directory @@ -3072,34 +3054,34 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download big size apps from Gcs + // Download big size apps from GCS appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big size app to Gcs for Indexer Cluster + // Upload big size app to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Indexer Cluster") + testcaseEnvInst.Log.Info("Upload big size app to GCS for Indexer Cluster") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Indexer Cluster") + Expect(err).To(Succeed(), "Unable to upload big size to GCS test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Search Head Cluster") + // Upload big size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big size to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3109,25 +3091,25 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3141,10 +3123,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -3155,7 +3138,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP ################## - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Prepare and deploy C3 CRD with app framework * Verify IsDeploymentInProgress is set * Wait for the pods to be ready @@ -3165,27 +3148,27 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3195,23 +3178,23 @@ var _ = Describe("c3appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) @@ -3252,16 +3235,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Ensure that the Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3272,7 +3255,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -3281,7 +3264,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload ES app to Gcs + * Upload ES app to GCS * Upload TA add-on app to location for Indexer cluster * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA * Prepare and deploy C3 CRD with app framework and wait for pods to be ready @@ -3289,46 +3272,46 @@ var _ = Describe("c3appfw test", func() { * Verify ES app is installed on Deployer and on Search Heads * Verify TA add-on app is installed on indexers ################## UPGRADE VERIFICATION ############# - * Update ES app on Gcs location + * Update ES app on GCS location * Verify updated ES app is installed on Deployer and on Search Heads */ //################## SETUP #################### - // Download ES app from Gcs + // Download ES app from GCS appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Gcs") + testcaseEnvInst.Log.Info("Download ES app from GCS") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download ES app file from GCS") - // Download Technology add-on app from Gcs - testcaseEnvInst.Log.Info("Download Technology add-on app from Gcs") + // Download Technology add-on app from GCS + testcaseEnvInst.Log.Info("Download Technology add-on app from GCS") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) + Expect(err).To(Succeed(), "Unable to download ES app file from GCS") - // Create directory for file upload to Gcs + // Create directory for file upload to GCS gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - // Upload ES app to Gcs - testcaseEnvInst.Log.Info("Upload ES app to Gcs") + // Upload ES app to GCS + testcaseEnvInst.Log.Info("Upload ES app to GCS") uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to Gcs test directory") + Expect(err).To(Succeed(), "Unable to upload ES app to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload Technology add-on apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to Gcs for Indexer Cluster", appVersion)) + // Upload Technology add-on apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileListIdxc, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -3336,10 +3319,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) // Deploy C3 SVA // Deploy the Cluster Manager @@ -3383,16 +3366,16 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -3402,29 +3385,31 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // //############### UPGRADE APPS ################ - // // Download ES App from Gcs + // // Download ES App from GCS // appVersion = "V2" - // testcaseEnvInst.Log.Info("Download updated ES app from Gcs") + // testcaseEnvInst.Log.Info("Download updated ES app from GCS") // err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") - // // Upload V2 ES app to Gcs for Search Head Cluster - // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Gcs for Search Head Cluster", appVersion)) + // // Upload V2 ES app to GCS for Search Head Cluster + // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to GCS for Search Head Cluster", appVersion)) // uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Gcs test directory for Search Head Cluster", appVersion)) + // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to GCS test directory for Search Head Cluster", appVersion)) // uploadedApps = append(uploadedApps, uploadedFiles...) // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) + // testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) // // Ensure that the Cluster Manager goes to Ready phase // testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_gcp/m4/appframework_gcs_suite_test.go b/test/appframework_gcp/m4/appframework_gcs_suite_test.go index 46a2c6b28..457f139c8 100644 --- a/test/appframework_gcp/m4/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -63,44 +49,10 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - testenvInstance.Log.Info("logging download details", "bucket", testDataGcsBucket, "gcsAppDirV1", gcsAppDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_gcp/m4/appframework_gcs_test.go b/test/appframework_gcp/m4/appframework_gcs_test.go index d13a8c455..b27c41a09 100644 --- a/test/appframework_gcp/m4/appframework_gcs_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -51,15 +50,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).ToNot(HaveOccurred()) gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -68,30 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { /* Test Steps @@ -132,10 +104,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -154,7 +126,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to GCP for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) @@ -168,11 +140,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -184,28 +156,19 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -217,7 +180,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //########## INITIAL VERIFICATIONS ########## var idxcPodNames, shcPodNames []string @@ -230,15 +193,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# UPGRADE APPS ################ // Delete apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // get revision number of the resource @@ -265,28 +229,19 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -302,10 +257,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -351,10 +307,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -373,7 +329,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to GCP for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) @@ -387,11 +343,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -400,25 +356,16 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -434,16 +381,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete V2 apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V1 apps to GCP for Indexer Cluster @@ -467,25 +415,16 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -501,10 +440,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -564,11 +504,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -579,20 +519,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -601,8 +532,8 @@ var _ = Describe("m4appfw test", func() { for i := 1; i <= siteCount; i++ { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") } // ############ Verify livenessProbe and readinessProbe config object and scripts############ @@ -620,10 +551,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -631,9 +563,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -646,13 +576,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -663,28 +592,28 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ingest data on new Indexers podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -702,7 +631,8 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -710,20 +640,18 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -736,11 +664,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -751,20 +678,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -781,10 +708,11 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -830,11 +758,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -845,14 +773,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -866,15 +788,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to GCP for Indexer Cluster @@ -892,16 +815,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -914,10 +831,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -965,10 +883,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -987,7 +905,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to GCP for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) @@ -1001,36 +919,27 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1046,10 +955,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ @@ -1074,30 +984,21 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1110,13 +1011,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1128,10 +1029,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1143,11 +1044,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) + err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 10*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1172,10 +1073,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1225,11 +1127,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -1239,14 +1141,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1260,15 +1156,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to GCP for Indexer Cluster @@ -1286,20 +1183,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1313,13 +1205,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1331,10 +1223,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1355,10 +1247,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1428,7 +1321,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1438,9 +1331,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1450,8 +1343,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1470,20 +1363,11 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1499,15 +1383,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1548,22 +1433,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ########## Verify Manual Poll config map disabled after the poll is triggered ################# @@ -1586,10 +1462,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1623,7 +1500,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1643,12 +1520,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1667,23 +1544,23 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to GCP for Cluster Master appList = testenv.ExtraApps @@ -1700,7 +1577,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -1710,10 +1587,10 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for SearchHeadCluster to reach Ready phase err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) @@ -1725,7 +1602,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Deployer deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) @@ -1760,7 +1637,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1780,12 +1657,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1804,13 +1681,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1818,10 +1695,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to GCP for Cluster Master appList = testenv.ExtraApps @@ -1838,23 +1715,23 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -1862,7 +1739,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -1889,7 +1766,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1905,11 +1782,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1917,28 +1794,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1952,10 +1820,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1981,7 +1850,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1997,11 +1866,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2009,28 +1878,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2044,10 +1904,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2087,11 +1948,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2099,22 +1960,13 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2127,28 +1979,29 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on GCS") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2159,7 +2012,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCP test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from GCP bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2205,11 +2058,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -2219,14 +2072,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2240,10 +2087,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") @@ -2253,8 +2101,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + Expect(deployment.GetInstance(ctx, deployment.GetName(), cm)).To(Succeed(), "Failed to get instance of Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2264,8 +2111,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2300,7 +2146,7 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to GCP for Indexer Cluster @@ -2323,7 +2169,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2341,10 +2188,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2380,7 +2228,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2395,11 +2243,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2407,10 +2255,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to GCP for Indexer Cluster appVersion = "V2" @@ -2432,26 +2280,26 @@ var _ = Describe("m4appfw test", func() { //########## VERIFICATIONS ########## appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2463,7 +2311,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2487,7 +2336,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2496,7 +2345,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2507,7 +2356,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) gcsTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) @@ -2516,14 +2365,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -2534,14 +2383,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + Expect(testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 Indexers and SHC not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2555,7 +2398,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2581,7 +2425,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2597,11 +2441,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2609,31 +2453,22 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2647,10 +2482,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2681,11 +2517,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2693,30 +2529,23 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)).To(Succeed(), "Cluster Master not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady)).To(Succeed(), "M4 cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) }) diff --git a/test/appframework_gcp/m4/manager_appframework_test.go b/test/appframework_gcp/m4/manager_appframework_test.go index e7b6ff142..9fe8f703b 100644 --- a/test/appframework_gcp/m4/manager_appframework_test.go +++ b/test/appframework_gcp/m4/manager_appframework_test.go @@ -21,7 +21,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -50,15 +49,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -67,30 +59,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { /* Test Steps @@ -131,10 +103,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -153,7 +125,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to GCP for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) @@ -167,11 +139,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -183,28 +155,28 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -216,7 +188,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //########## INITIAL VERIFICATIONS ########## var idxcPodNames, shcPodNames []string @@ -229,15 +201,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# UPGRADE APPS ################ // Delete apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // get revision number of the resource @@ -264,28 +237,28 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -301,10 +274,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -350,10 +324,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -372,7 +346,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V2 apps to GCP for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) @@ -386,11 +360,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -399,25 +373,25 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -433,16 +407,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############# DOWNGRADE APPS ################ // Delete V2 apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V1 apps to GCP for Indexer Cluster @@ -466,25 +441,25 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -500,10 +475,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -563,11 +539,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -579,19 +555,19 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -600,8 +576,8 @@ var _ = Describe("m4appfw test", func() { for i := 1; i <= siteCount; i++ { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") } // ############ Verify livenessProbe and readinessProbe config object and scripts############ @@ -619,10 +595,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -630,9 +607,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -645,13 +620,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -662,28 +636,28 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ingest data on new Indexers podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Failed to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName)).To(Succeed(), "Failed to ingest file via monitor") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -701,7 +675,8 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -709,20 +684,18 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -735,11 +708,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Search Head Cluster phase mismatch") // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, idxcName, idxc)).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -750,20 +722,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName)).To(Succeed(), "Indexer Cluster phase mismatch") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -780,10 +752,11 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -829,11 +802,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -845,13 +818,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -865,15 +838,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to GCP for Indexer Cluster @@ -891,16 +865,16 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -913,10 +887,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -964,10 +939,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -986,7 +961,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to GCP for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) @@ -1000,36 +975,36 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1045,10 +1020,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // ############### UPGRADE APPS ################ @@ -1073,30 +1049,30 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1109,13 +1085,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1127,10 +1103,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1142,11 +1118,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) + err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 10*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1171,10 +1147,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1224,11 +1201,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1239,13 +1216,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1259,15 +1236,16 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete V1 apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to GCP for Indexer Cluster @@ -1285,20 +1263,21 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1312,13 +1291,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") testcaseEnvInst.Log.Info("Get config map for triggering manual update") config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) @@ -1330,10 +1309,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1354,10 +1333,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1427,7 +1407,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1437,9 +1417,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1449,8 +1429,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1470,19 +1450,19 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1498,15 +1478,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") //############### UPGRADE APPS ################ // Delete apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1547,22 +1528,22 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // ########## Verify Manual Poll config map disabled after the poll is triggered ################# @@ -1585,10 +1566,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1622,7 +1604,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1642,12 +1624,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1666,23 +1648,23 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to GCP for Cluster Manager appList = testenv.ExtraApps @@ -1699,7 +1681,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) @@ -1709,10 +1691,10 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for SearchHeadCluster to reach Ready phase err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) @@ -1723,7 +1705,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Deployer deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) @@ -1758,7 +1740,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1778,12 +1760,12 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1802,13 +1784,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1816,10 +1798,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to GCP for Cluster Manager appList = testenv.ExtraApps @@ -1836,23 +1818,23 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -1860,7 +1842,7 @@ var _ = Describe("m4appfw test", func() { // Verify all apps are installed on Search Heads shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true)).To(Succeed(), "App installation verification failed") }) }) @@ -1887,7 +1869,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1903,11 +1885,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1915,28 +1897,28 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1950,10 +1932,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -1979,7 +1962,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1995,11 +1978,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2007,28 +1990,28 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2042,10 +2025,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2085,11 +2069,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2097,22 +2081,22 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2125,28 +2109,29 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToGCP(downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on GCS") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) @@ -2157,7 +2142,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCP test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from GCP bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -2203,11 +2188,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2218,13 +2203,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2238,10 +2223,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") // Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster") @@ -2251,8 +2237,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Manager CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") + Expect(deployment.GetInstance(ctx, deployment.GetName(), cm)).To(Succeed(), "Failed to get instance of Cluster Manager") // Set AppsRepoPollInterval for Cluster Manager to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") @@ -2262,8 +2247,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc)).To(Succeed(), "Failed to get instance of Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2298,7 +2282,7 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on GCP testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to GCP for Indexer Cluster @@ -2321,7 +2305,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2339,10 +2324,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2378,7 +2364,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2393,11 +2379,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2405,10 +2391,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to GCP for Indexer Cluster appVersion = "V2" @@ -2430,26 +2416,26 @@ var _ = Describe("m4appfw test", func() { //########## VERIFICATIONS ########## appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") //############ UPGRADE VERIFICATIONS ############ appVersion = "V2" @@ -2461,7 +2447,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2485,7 +2472,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2494,7 +2481,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2505,7 +2492,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) @@ -2514,14 +2501,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2533,13 +2520,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2553,7 +2540,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2579,7 +2567,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2595,11 +2583,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2607,31 +2595,31 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2645,10 +2633,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + Expect(testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil)).To(Succeed(), "Unexpected pod reset detected") }) }) @@ -2679,11 +2668,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2691,30 +2680,30 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment)).To(Succeed(), "Cluster Manager not ready") // Verify IsDeploymentInProgress Flag is set to true for SHC CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Indexers not ready") // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)).To(Succeed(), "Indexer Cluster multisite status verification failed") // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") }) }) }) diff --git a/test/appframework_gcp/s1/appframework_gcs_suite_test.go b/test/appframework_gcp/s1/appframework_gcs_suite_test.go index 97f2d6969..3cff417ee 100644 --- a/test/appframework_gcp/s1/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGCSBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -63,39 +49,10 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_gcp/s1/appframework_gcs_test.go b/test/appframework_gcp/s1/appframework_gcs_test.go index d1698d539..0fa240b63 100644 --- a/test/appframework_gcp/s1/appframework_gcs_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -45,42 +44,15 @@ var _ = Describe("s1appfw test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGCSBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -128,10 +100,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -150,7 +122,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to gcs for Standalone @@ -162,9 +134,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -183,13 +155,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -201,7 +173,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") // ############ INITIAL VERIFICATION ########### standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} @@ -209,13 +181,14 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# // Delete apps on gcs testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to gcs for Standalone and Monitoring Console @@ -232,13 +205,13 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -251,7 +224,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -308,10 +282,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -330,11 +304,11 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -352,13 +326,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -369,12 +343,13 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############# DOWNGRADE APPS ################ // Delete apps on gcs testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // get revision number of the resource @@ -394,16 +369,16 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + Expect(testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion)).To(Succeed(), "Custom resource version not changed") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -416,7 +391,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -479,10 +455,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -509,9 +485,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -529,13 +505,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -553,7 +529,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -564,8 +541,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -573,16 +549,17 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady)).To(Succeed(), "Standalone phase mismatch") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") //########### SCALING UP VERIFICATION ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -590,31 +567,31 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") //############## SCALING DOWN ################# // Scale down Standalone instance testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) Expect(err).To(Succeed(), "Failed to scale down Standalone") // Ensure Standalone is scaling down - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingDown) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingDown)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady)).To(Succeed(), "Standalone phase mismatch") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") //########### SCALING DOWN VERIFICATION ####### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -667,9 +644,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -684,10 +661,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -697,15 +674,15 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### SCALING UP ################## // Scale up Standalone instance testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -713,15 +690,15 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed(), "Standalone phase mismatch") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // ############## UPGRADE APPS ################# // Delete apps on gcs testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to gcs for Standalone and Monitoring Console @@ -734,10 +711,10 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -748,7 +725,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -771,7 +749,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from gcs") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to gcs @@ -780,9 +758,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -803,10 +781,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Ensure Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -816,18 +794,19 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# // Delete apps on gcs testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Download ES App from gcs testcaseEnvInst.Log.Info("Download updated ES app from gcs") - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to gcs for Standalone @@ -838,10 +817,10 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -851,7 +830,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = esApp standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -882,7 +862,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from gcs testcaseEnvInst.Log.Info("Download bigger amount of apps from gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -892,9 +872,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -912,7 +892,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -921,7 +901,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -972,10 +953,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -994,16 +975,16 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Upload V1 apps to GCS uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1021,13 +1002,13 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1036,12 +1017,13 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ //Delete apps on GCS for new Apps - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil //Upload new Versioned Apps to GCS @@ -1057,17 +1039,18 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1083,10 +1066,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1101,7 +1084,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1134,7 +1118,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from GCS testcaseEnvInst.Log.Info("Download the extra apps from GCS for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to GCS for first Standalone @@ -1152,9 +1136,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1166,10 +1150,10 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) + appFrameworkSpecStandalone2 := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) specStandalone2 := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1191,8 +1175,8 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standalone2Name, standalone2) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standalone2Name, standalone2)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1204,12 +1188,13 @@ var _ = Describe("s1appfw test", func() { standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can add new apps to app source while install is in progress and have all apps installed", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integrationgcp, s1gcp, appframeworkgcp: can add new apps to app source while install is in progress and have all apps installed", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1236,10 +1221,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1258,13 +1243,13 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Download all test apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1275,9 +1260,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1295,10 +1280,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload more apps to GCS for Standalone appList = testenv.ExtraApps @@ -1309,7 +1294,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) @@ -1318,12 +1303,12 @@ var _ = Describe("s1appfw test", func() { appList = append(testenv.BigSingleApp, testenv.ExtraApps...) standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false)).To(Succeed(), "App installation verification failed") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1343,7 +1328,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1352,9 +1337,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1369,7 +1354,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1381,17 +1366,17 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1400,20 +1385,24 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) + err = wait.PollUntilContextTimeout(ctx, testenv.PollInterval, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + _, getErr := testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) + return getErr == nil, nil + }) Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true)).To(Succeed(), "Files not found in directory on pod") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1433,7 +1422,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1442,9 +1431,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1459,16 +1448,16 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + Expect(testcaseEnvInst.DeleteOperatorPod()).To(Succeed(), "Failed to delete operator pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1477,7 +1466,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1515,9 +1505,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1533,10 +1523,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1545,22 +1535,23 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0])).To(Succeed(), "App repo state verification failed") // Disable the app err = testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDir) Expect(err).To(Succeed(), "Unable to disable apps on GCS") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileName) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Wait for App state to update after config file change standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) @@ -1572,7 +1563,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCS test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from GCS bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) + Expect(testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0])).To(Succeed(), "App repo state verification failed") }) }) @@ -1595,7 +1586,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to GCS App Source @@ -1626,9 +1617,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1645,7 +1636,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) Expect(err).To(Succeed(), "Unable to obtain secret object") @@ -1653,7 +1644,7 @@ var _ = Describe("s1appfw test", func() { modifiedSecretData := map[string][]byte{"gcs_access_key": []byte(testenv.RandomDNSName(5)), "gcs_secret_key": []byte(testenv.RandomDNSName(5))} // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1662,10 +1653,11 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1673,7 +1665,7 @@ var _ = Describe("s1appfw test", func() { // ############## UPGRADE APPS ################# // Delete apps on testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) + Expect(testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps)).To(Succeed(), "GCP file deletion failed") uploadedApps = nil // Upload V2 apps to GCS for Standalone @@ -1686,10 +1678,11 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Check no apps are updated as auth key is incorrect - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key to correct one########### // Apply spec with correct credentials @@ -1697,10 +1690,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to update secret Object") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1710,13 +1703,14 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1738,7 +1732,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to GCS for Standalone @@ -1747,9 +1741,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1764,10 +1758,10 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") // Upload V2 apps to GCS for Standalone appVersion = "V2" @@ -1780,13 +1774,13 @@ var _ = Describe("s1appfw test", func() { //######### VERIFICATIONS ############# appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) + Expect(testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false)).To(Succeed(), "App installation verification failed") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1796,7 +1790,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1821,7 +1816,7 @@ var _ = Describe("s1appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -1830,7 +1825,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCS @@ -1842,9 +1837,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1863,7 +1858,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Standalone instance") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1872,12 +1867,13 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { /* Test Steps ################## SETUP #################### @@ -1897,7 +1893,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to GCS for Standalone @@ -1906,9 +1902,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1923,19 +1919,19 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) + Expect(testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending, testenv.AppStateVerificationTimeout)).To(Succeed(), "App state verification failed") //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1944,7 +1940,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1977,10 +1974,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -2000,10 +1997,10 @@ var _ = Describe("s1appfw test", func() { // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, mcName, mc.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, mcName, mc.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to GCS for Standalone @@ -2015,9 +2012,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2036,16 +2033,16 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) + Expect(testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind)).To(Succeed(), "IsDeploymentInProgress flag not set") // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") }) }) diff --git a/test/custom_resource_crud/crud_test_shared.go b/test/custom_resource_crud/crud_test_shared.go new file mode 100644 index 000000000..01cdadc15 --- /dev/null +++ b/test/custom_resource_crud/crud_test_shared.go @@ -0,0 +1,253 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunS1CPUUpdateTest runs the standard S1 CPU limit update test workflow +func RunS1CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string, newCPULimits string) { + // Deploy and verify Standalone + standalone, err := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") + + // Verify telemetry + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + Expect(testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime)).To(Succeed(), "Telemetry verification failed") + + // Deploy and verify Monitoring Console + mcRef := deployment.GetName() + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Verify CPU limits on Standalone before updating the CR + standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + Expect(testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits)).To(Succeed(), "Standalone CPU limits mismatch before CR update") + + // Change CPU limits to trigger CR update + standalone.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + err = deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Unable to update Standalone CR") + + // Verify Standalone reaches Updating phase and returns to Ready + Expect(testcaseEnvInst.VerifyStandalonePhaseAndReady(ctx, deployment, enterpriseApi.PhaseUpdating, standalone)).To(Succeed(), "Standalone did not reach Updating phase or return to Ready") + + // Verify Monitoring Console is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcRef, mc)).To(Succeed(), "Monitoring Console not ready") + + // Verify CPU limits on Standalone after updating the CR + Expect(testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits)).To(Succeed(), "Standalone CPU limits mismatch after CR update") +} + +// RunC3CPUUpdateTest runs the standard C3 CPU limit update test workflow +func RunC3CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + Expect(config.DeployAndVerifyC3(ctx, deployment, testcaseEnvInst, 3, true /*shc*/, mcRef)).To(Succeed(), "Unable to deploy C3 cluster") + + // Verify telemetry + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + Expect(testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime)).To(Succeed(), "Telemetry verification failed") + + // Deploy and verify Monitoring Console, RF/SF + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + Expect(testcaseEnvInst.StandardC3Verification(ctx, deployment, mc)).To(Succeed(), "Standard C3 verification failed") + + // Verify CPU limits on Indexers before updating the CR + indexerCount := 3 + Expect(testcaseEnvInst.VerifyIndexerCPULimits(deployment, indexerCount, defaultCPULimits)).To(Succeed(), "Indexer CPU limits mismatch before CR update") + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) + Expect(deployment.GetInstance(ctx, instanceName, idxc)).To(Succeed(), "Unable to get Indexer Cluster instance") + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + Expect(deployment.UpdateCR(ctx, idxc)).To(Succeed(), "Unable to update Indexer Cluster CR") + + // Verify Indexer Cluster is updating + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, instanceName)).To(Succeed(), "Indexer Cluster did not reach Updating phase") + + // Verify Indexers go to ready state + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready after CR update") + + // Verify CPU limits on Indexers after updating the CR + Expect(testcaseEnvInst.VerifyIndexerCPULimits(deployment, indexerCount, newCPULimits)).To(Succeed(), "Indexer CPU limits mismatch after CR update") + + // Verify CPU limits on Search Heads before updating the CR + searchHeadCount := 3 + Expect(testcaseEnvInst.VerifySearchHeadCPULimits(deployment, searchHeadCount, defaultCPULimits)).To(Succeed(), "Search Head CPU limits mismatch before CR update") + + // Change CPU limits to trigger CR update + shc := &enterpriseApi.SearchHeadCluster{} + instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) + Expect(deployment.GetInstance(ctx, instanceName, shc)).To(Succeed(), "Unable to get Search Head Cluster instance") + + shc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + Expect(deployment.UpdateCR(ctx, shc)).To(Succeed(), "Unable to update Search Head Cluster CR") + + // Verify Search Head Cluster is updating + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating)).To(Succeed(), "Search Head Cluster did not reach Updating phase") + + // Verify Search Heads go to ready state + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready after CR update") + + // Verify Monitoring Console is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcRef, mc)).To(Succeed(), "Monitoring Console not ready") + + // Verify CPU limits on Search Heads after updating the CR + Expect(testcaseEnvInst.VerifySearchHeadCPULimits(deployment, searchHeadCount, newCPULimits)).To(Succeed(), "Search Head CPU limits mismatch after CR update") +} + +// RunC3PVCDeletionTest runs the standard C3 PVC deletion test workflow +func RunC3PVCDeletionTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, verificationTimeout time.Duration) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + Expect(config.DeployAndVerifyC3(ctx, deployment, testcaseEnvInst, 3, true /*shc*/, mcRef)).To(Succeed(), "Unable to deploy C3 cluster") + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + clusterManagerType := config.ClusterManagerPVCType() + Expect(testenv.VerifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, true, verificationTimeout)).To(Succeed(), "C3 cluster PVCs not present") + + // Delete the Search Head Cluster + Expect(testenv.GetAndDeleteCR(ctx, deployment, &enterpriseApi.SearchHeadCluster{}, deployment.GetName()+"-shc")).To(Succeed(), "Unable to delete SHC instance") + + // Delete the Indexer Cluster + Expect(testenv.GetAndDeleteCR(ctx, deployment, &enterpriseApi.IndexerCluster{}, deployment.GetName()+"-idxc")).To(Succeed(), "Unable to delete IDXC instance") + + // Delete the Cluster Manager (v3 or v4) + Expect(config.DeleteClusterManager(ctx, deployment)).To(Succeed(), "Unable to delete Cluster Manager") + + // Delete Monitoring Console + Expect(testenv.GetAndDeleteCR(ctx, deployment, mc, mcRef)).To(Succeed(), "Unable to delete Monitoring Console instance") + Expect(testenv.VerifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, false, verificationTimeout)).To(Succeed(), "C3 cluster PVCs not deleted") + + // Verify Monitoring Console PVCs (etc and var) have been deleted + Expect(testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout)).To(Succeed(), "Monitoring Console PVCs not deleted") +} + +// RunSHCDeployerResourceSpecTest deploys a Search Head Cluster, verifies default CPU limits, +// updates the deployer resource spec, and verifies the deployer is reconfigured while search heads retain defaults. +func RunSHCDeployerResourceSpecTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string) { + shcName := fmt.Sprintf("%s-shc", deployment.GetName()) + _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") + Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "shc", shcName) + + // Verify CPU limits on Search Heads and deployer before updating CR + searchHeadCount := 3 + Expect(testcaseEnvInst.VerifySearchHeadCPULimits(deployment, searchHeadCount, defaultCPULimits)).To(Succeed(), "Search Head CPU limits mismatch before CR update") + + deployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) + Expect(testcaseEnvInst.VerifyCPULimits(deployment, deployerPodName, defaultCPULimits)).To(Succeed(), "Deployer CPU limits mismatch before CR update") + + shc := &enterpriseApi.SearchHeadCluster{} + Expect(deployment.GetInstance(ctx, shcName, shc)).To(Succeed(), "Unable to get Search Head Cluster instance") + + // Assign new resources for deployer pod only + newCPULimits := "4" + newCPURequests := "2" + newMemoryLimits := "14Gi" + newMemoryRequests := "12Gi" + + depResSpec := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse(newCPURequests), + "memory": resource.MustParse(newMemoryRequests), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + "memory": resource.MustParse(newMemoryLimits), + }, + } + shc.Spec.DeployerResourceSpec = depResSpec + Expect(deployment.UpdateCR(ctx, shc)).To(Succeed(), "Unable to update Search Head Cluster CR") + + // Verify Search Heads go to ready state + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready after deployer spec update") + + // Verify CPU limits on Search Heads - Should be same as before + Expect(testcaseEnvInst.VerifySearchHeadCPULimits(deployment, searchHeadCount, defaultCPULimits)).To(Succeed(), "Search Head CPU limits changed unexpectedly") + + // Verify modified deployer spec + Expect(testcaseEnvInst.VerifyResourceConstraints(deployment, deployerPodName, depResSpec)).To(Succeed(), "Deployer resource constraints mismatch") +} + +// RunM4CPUUpdateTest runs the standard M4 CPU limit update test workflow +func RunM4CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Multisite Cluster and Search Head Clusters + mcRef := deployment.GetName() + siteCount := 3 + Expect(config.DeployAndVerifyM4(ctx, deployment, testcaseEnvInst, 1, siteCount, mcRef)).To(Succeed(), "Unable to deploy M4 cluster") + + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + Expect(testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime)).To(Succeed(), "Telemetry verification failed") + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Verify RF SF is met + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met") + + // Verify CPU limits on Indexers before updating the CR + Expect(testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, siteCount, defaultCPULimits)).To(Succeed(), "Multisite Indexer CPU limits mismatch before CR update") + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + for i := 1; i <= siteCount; i++ { + siteName := fmt.Sprintf("site%d", i) + instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) + Expect(deployment.GetInstance(ctx, instanceName, idxc)).To(Succeed(), "Unable to get Indexer Cluster instance") + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + Expect(deployment.UpdateCR(ctx, idxc)).To(Succeed(), "Unable to update Indexer Cluster CR") + } + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-site1" + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName)).To(Succeed(), "Indexer Cluster did not reach Updating phase") + + // Verify Indexers go to ready state + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed(), "Multisite Indexers not ready after CR update") + + // Verify Monitoring Console is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "Monitoring Console not ready") + + // Verify RF SF is met + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed(), "RF/SF not met after CR update") + + // Verify CPU limits after updating the CR + Expect(testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, siteCount, newCPULimits)).To(Succeed(), "Multisite Indexer CPU limits mismatch after CR update") +} diff --git a/test/custom_resource_crud/custom_resource_crud_c3_test.go b/test/custom_resource_crud/custom_resource_crud_c3_test.go deleted file mode 100644 index f1f12dec0..000000000 --- a/test/custom_resource_crud/custom_resource_crud_c3_test.go +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Master - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_m4_test.go b/test/custom_resource_crud/custom_resource_crud_m4_test.go deleted file mode 100644 index b057692be..000000000 --- a/test/custom_resource_crud/custom_resource_crud_m4_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_s1_test.go b/test/custom_resource_crud/custom_resource_crud_s1_test.go deleted file mode 100644 index 80a5e1c34..000000000 --- a/test/custom_resource_crud/custom_resource_crud_s1_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1)", func() { - It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - - // Deploy Standalone - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits before updating the CR - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits) - - // Change CPU limits to trigger CR update - standalone.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") - - // Verify Standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits after updating the CR - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_suite_test.go b/test/custom_resource_crud/custom_resource_crud_suite_test.go index 6dffc9788..b45c94903 100644 --- a/test/custom_resource_crud/custom_resource_crud_suite_test.go +++ b/test/custom_resource_crud/custom_resource_crud_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,12 +24,12 @@ import ( ) const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond + // DefaultCPULimits is the default CPU limit + DefaultCPULimits = "4" + // UpdatedCPULimits is the updated CPU limit + UpdatedCPULimits = "2" + // DefaultVerificationTimeout is the default timeout for CRUD verification steps + DefaultVerificationTimeout = 150 * time.Second ) var ( @@ -37,9 +37,8 @@ var ( testSuiteName = "crcrud-" + testenv.RandomDNSName(3) ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - +// TestCRCRUD is the main entry point +func TestCRCRUD(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -51,11 +50,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/custom_resource_crud/custom_resource_crud_test.go b/test/custom_resource_crud/custom_resource_crud_test.go new file mode 100644 index 000000000..d68de91c9 --- /dev/null +++ b/test/custom_resource_crud/custom_resource_crud_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// masterManagerCrudConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 and M4 CRUD test tables. +var masterManagerCrudConfigs = []testenv.MasterManagerTestConfig{ + {NamePrefix: "master", Label: "mastercrcrud", NewConfig: testenv.NewClusterReadinessConfigV3}, + {NamePrefix: "", Label: "managercrcrud", NewConfig: testenv.NewClusterReadinessConfigV4}, +} + +var _ = Describe("Custom Resource CRUD test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var defaultCPULimits string + var newCPULimits string + var verificationTimeout time.Duration + + ctx := context.TODO() + + // S1 test — single variant (manager, V4) + Context("Standalone deployment (S1)", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It("managercrcrud, integration, s1: can deploy a Standalone instance, change its CR, update the instance", func() { + RunS1CPUUpdateTest(ctx, deployment, testcaseEnvInst, defaultCPULimits, newCPULimits) + }) + }) + + // C3 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerCrudConfigs { + tc := tc + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + verificationTimeout = DefaultVerificationTimeout + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, c3: can deploy Indexer and Search Head Cluster, change their CR, update the instances", func() { + config := tc.NewConfig() + RunC3CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + + It(tc.Label+", integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { + config := tc.NewConfig() + RunC3PVCDeletionTest(ctx, deployment, testcaseEnvInst, config, verificationTimeout) + }) + }) + } + + // CSPL-3256 - SHC deployer resource spec test (IDXC is irrelevant for this test case) + Context("Search Head Cluster", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { + RunSHCDeployerResourceSpecTest(ctx, deployment, testcaseEnvInst, defaultCPULimits) + }) + }) + + // M4 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerCrudConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, m4: can deploy multisite Indexer and Search Head Clusters, change their CR, update the instances", func() { + config := tc.NewConfig() + RunM4CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + }) + } +}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go b/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go deleted file mode 100644 index a2b9becc1..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Search Head Cluster", func() { - // CSPL-3256 - Adding the SHC only test case under c3 as IDXC is irrelevant for this test case - It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - shcName := fmt.Sprintf("%s-shc", deployment.GetName()) - _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") - if err != nil { - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "Shc", shcName) - } - - // Verify CPU limits on Search Heads and deployer before updating CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - DeployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) - testcaseEnvInst.VerifyCPULimits(deployment, DeployerPodName, defaultCPULimits) - - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - // Assign new resources for deployer pod only - newCPULimits = "4" - newCPURequests := "2" - newMemoryLimits := "14Gi" - newMemoryRequests := "12Gi" - - depResSpec := corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": resource.MustParse(newCPURequests), - "memory": resource.MustParse(newMemoryRequests), - }, - Limits: corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - "memory": resource.MustParse(newMemoryLimits), - }, - } - shc.Spec.DeployerResourceSpec = depResSpec - - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify CPU limits on Search Heads - Should be same as before - searchHeadCount = 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Verify modified deployer spec - testcaseEnvInst.VerifyResourceConstraints(deployment, DeployerPodName, depResSpec) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Manager - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go b/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go deleted file mode 100644 index aaee913e3..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/delete_cr/deletecr_suite_test.go b/test/delete_cr/deletecr_suite_test.go index 75c3b2abb..6941e3353 100644 --- a/test/delete_cr/deletecr_suite_test.go +++ b/test/delete_cr/deletecr_suite_test.go @@ -1,23 +1,20 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,23 +22,13 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "deletecr-" + testenv.RandomDNSName(3) ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - +// TestDeleteCR is the main entry point +func TestDeleteCR(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -53,11 +40,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/delete_cr/deletecr_test.go b/test/delete_cr/deletecr_test.go index 765f029b6..6a80d3b7b 100644 --- a/test/delete_cr/deletecr_test.go +++ b/test/delete_cr/deletecr_test.go @@ -1,136 +1,49 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( - "fmt" - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" + "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("DeleteCR test", func() { +var _ = Describe("Delete CR test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) - Context("Standalone deployment (S1 - Standalone Pod)", func() { + Context("Standalone deployment (S1)", func() { It("integration, managerdeletecr: can deploy standalone and delete", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Delete Standalone CR - err = deployment.DeleteCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to Delete Standalone") - + Expect(testcaseEnvInst.RunDeleteStandaloneWorkflow(ctx, deployment)).To(Succeed(), "Unable to run delete Standalone workflow") }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3)", func() { It("integration, managerdeletecr: can deploy C3 and delete search head, clustermanager", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Deploy C3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, true, "") - Expect(err).To(Succeed(), "Unable to deploy C3 instance") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - idxc := &enterpriseApi.IndexerCluster{} - idxcName := deployment.GetName() + "-idxc" - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Unable to get Indexer instance") - - // Delete Indexer Cluster CR - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to Delete Indexer Cluster") - - sh := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, sh) - Expect(err).To(Succeed(), "Unable to get Search Head instance") - - // Delete Search Head Cluster CR - err = deployment.DeleteCR(ctx, sh) - Expect(err).To(Succeed(), "Unable to Delete Search Head Cluster") - - cm := &enterpriseApi.ClusterManager{} - cmName := deployment.GetName() - err = deployment.GetInstance(ctx, cmName, cm) - Expect(err).To(Succeed(), "Unable to get Cluster Manager instance") - - // Delete Cluster Manager CR - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to Delete Cluster Manager") - + Expect(testcaseEnvInst.RunDeleteC3Workflow(ctx, deployment, 3)).To(Succeed(), "Unable to run delete C3 workflow") }) }) }) diff --git a/test/example/example1_test.go b/test/example/example1_test.go index bba0023d1..b2dd4be29 100644 --- a/test/example/example1_test.go +++ b/test/example/example1_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +14,6 @@ package example import ( - "fmt" "math/rand" "time" @@ -32,19 +31,12 @@ var _ = XDescribe("Example1", func() { // This is invoke for each "It" spec below BeforeEach(func() { var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) // "It" spec diff --git a/test/example/example2_test.go b/test/example/example2_test.go index 3988e0976..b68dceaa2 100644 --- a/test/example/example2_test.go +++ b/test/example/example2_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +14,6 @@ package example import ( - "fmt" "math/rand" "time" @@ -32,19 +31,12 @@ var _ = XDescribe("Example2", func() { // This is invoke for each "It" spec below BeforeEach(func() { var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) // "It" spec diff --git a/test/example/example_suite_test.go b/test/example/example_suite_test.go index 7c5227a45..515895e74 100644 --- a/test/example/example_suite_test.go +++ b/test/example/example_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,9 +14,7 @@ package example import ( - "math/rand" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -29,12 +27,7 @@ var ( testSuiteName = "example-" + testenv.RandomDNSName(3) ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - func TestExampleSuite(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -46,9 +39,11 @@ func TestExampleSuite(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + if testenvInstance != nil { + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") + } }) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go index c25f3716e..732bf5b50 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go @@ -11,13 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,18 +25,9 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv - testSuiteName = "indingsep-" + testenv.RandomDNSName(3) + testSuiteName = "idxingsep-" + testenv.RandomDNSName(3) queue = enterpriseApi.QueueSpec{ Provider: "sqs", @@ -100,11 +90,10 @@ var ( appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) s3TestDir = "icappfw-" + testenv.RandomDNSName(4) appListV1 = testenv.BasicApps - s3AppDirV1 = testenv.AppLocationV1 ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { +// TestIndexIngestionSeparation is the main entry point +func TestIndexIngestionSeparation(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -116,19 +105,19 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") appListV1 = testenv.BasicApps appFileList := testenv.GetAppFileList(appListV1) // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download V1 app files") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } err := os.RemoveAll(downloadDirV1) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go index 697d9664b..77f75cd45 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "fmt" @@ -20,32 +20,24 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/onsi/ginkgo/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("indingsep test", func() { +var _ = Describe("Index and Ingestion Separation test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var cmSpec enterpriseApi.ClusterManagerSpec BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") // Validate test prerequisites early to fail fast err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) @@ -62,101 +54,22 @@ var _ = Describe("indingsep test", func() { }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to get Indexer Cluster instance", "Indexer Cluster Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete Indexer Cluster instance", "Indexer Cluster Name", idxc) + Expect(testcaseEnvInst.SetupIngestorStack(ctx, deployment, queue, objectStorage, cmSpec)).To(Succeed(), "Unable to setup ingestor stack") - // Delete the Ingestor Cluster - ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Unable to get Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - err = deployment.DeleteCR(ctx, ingest) - Expect(err).To(Succeed(), "Unable to delete Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - - // Delete the Queue - q = &enterpriseApi.Queue{} - err = deployment.GetInstance(ctx, "queue", q) - Expect(err).To(Succeed(), "Unable to get Queue instance", "Queue Name", q) - err = deployment.DeleteCR(ctx, q) - Expect(err).To(Succeed(), "Unable to delete Queue", "Queue Name", q) - - // Delete the ObjectStorage - objStorage = &enterpriseApi.ObjectStorage{} - err = deployment.GetInstance(ctx, "os", objStorage) - Expect(err).To(Succeed(), "Unable to get ObjectStorage instance", "ObjectStorage Name", objStorage) - err = deployment.DeleteCR(ctx, objStorage) - Expect(err).To(Succeed(), "Unable to delete ObjectStorage", "ObjectStorage Name", objStorage) + Expect(testenv.DeleteIngestorStack(ctx, deployment)).To(Succeed(), "Unable to delete ingestor stack") }) - }) - Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers with additional configurations", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers with additional configurations", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") @@ -169,19 +82,13 @@ var _ = Describe("indingsep test", func() { )} queue.SQS.VolList = volumeSpec - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") + // Deploy Queue and ObjectStorage + q, objStorage, err := testenv.DeployQueueAndObjectStorage(ctx, deployment, queue, objectStorage) + Expect(err).To(Succeed(), "Unable to deploy Queue and ObjectStorage") // Deploy Ingestor Cluster with additional configurations (similar to standalone app framework test) appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(5) ic := &enterpriseApi.IngestorCluster{ ObjectMeta: metav1.ObjectMeta{ @@ -230,7 +137,7 @@ var _ = Describe("indingsep test", func() { // Ensure that Ingestor Cluster is in Ready phase testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) + Expect(testcaseEnvInst.VerifyIngestorReady(ctx, deployment)).To(Succeed(), "Ingestor Cluster not ready") // Upload apps to S3 testcaseEnvInst.Log.Info("Upload apps to S3") @@ -254,92 +161,38 @@ var _ = Describe("indingsep test", func() { } allAppSourceInfo := []testenv.AppSourceInfo{ingestorAppSourceInfo} splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify probe configuration - testcaseEnvInst.Log.Info("Get config map for probes") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for probes", "ConfigMap", ConfigMapName) - testcaseEnvInst.Log.Info("Verify probe configurations on Ingestor pods") - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, true)).To(Succeed(), "Probe config verification failed") }) - }) - Context("Ingestor and Indexer deployment", func() { - It("indingsep, integration, indingsep: Splunk Operator can deploy Ingestors and Indexers with correct setup", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + It("indexingestionsep, integration: Splunk Operator can deploy Ingestors and Indexers with correct setup", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.SetupIngestorStack(ctx, deployment, queue, objectStorage, cmSpec)).To(Succeed(), "Unable to setup ingestor stack") // Get instance of current Ingestor Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Ingestor Cluster CR with latest config") ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Failed to get instance of Ingestor Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest)).To(Succeed(), "Failed to get instance of Ingestor Cluster") // Verify Ingestor Cluster Status testcaseEnvInst.Log.Info("Verify Ingestor Cluster Status") - Expect(ingest.Status.CredentialSecretVersion).To(Not(Equal("")), "Ingestor queue status credential access secret version is empty") - Expect(ingest.Status.CredentialSecretVersion).To(Not(Equal("0")), "Ingestor queue status credential access secret version is 0") + Expect(testenv.VerifyCredentialSecretVersion(ingest.Status.CredentialSecretVersion, "Ingestor")).To(Succeed(), "Ingestor credential secret version invalid") // Get instance of current Indexer Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Indexer Cluster CR with latest config") index := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", index) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + Expect(deployment.GetInstance(ctx, deployment.GetName()+"-idxc", index)).To(Succeed(), "Failed to get instance of Indexer Cluster") // Verify Indexer Cluster Status testcaseEnvInst.Log.Info("Verify Indexer Cluster Status") - Expect(index.Status.CredentialSecretVersion).To(Not(Equal("")), "Indexer queue status credential access secret version is empty") - Expect(index.Status.CredentialSecretVersion).To(Not(Equal("0")), "Indexer queue status credential access secret version is 0") + Expect(testenv.VerifyCredentialSecretVersion(index.Status.CredentialSecretVersion, "Indexer")).To(Succeed(), "Indexer credential secret version invalid") // Verify conf files testcaseEnvInst.Log.Info("Verify conf files") @@ -349,37 +202,25 @@ var _ = Describe("indingsep test", func() { if strings.Contains(pod, "ingest") || strings.Contains(pod, "idxc") { // Verify outputs.conf - testcaseEnvInst.Log.Info("Verify outputs.conf") - outputsPath := "opt/splunk/etc/system/local/outputs.conf" - outputsConf, err := testenv.GetConfFile(pod, outputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get outputs.conf from Ingestor Cluster pod") - testenv.ValidateContent(outputsConf, outputs, true) + Expect(testenv.VerifyConfFileContent(pod, "opt/splunk/etc/system/local/outputs.conf", deployment.GetName(), outputs, "Failed to get outputs.conf from Ingestor Cluster pod")).To(Succeed(), "outputs.conf verification failed") // Verify default-mode.conf - testcaseEnvInst.Log.Info("Verify default-mode.conf") - defaultsPath := "opt/splunk/etc/system/local/default-mode.conf" - defaultsConf, err := testenv.GetConfFile(pod, defaultsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get default-mode.conf from Ingestor Cluster pod") - testenv.ValidateContent(defaultsConf, defaultsAll, true) + Expect(testenv.VerifyConfFileContent(pod, "opt/splunk/etc/system/local/default-mode.conf", deployment.GetName(), defaultsAll, "Failed to get default-mode.conf from Ingestor Cluster pod")).To(Succeed(), "default-mode.conf verification failed") // Verify AWS env variables testcaseEnvInst.Log.Info("Verify AWS env variables") envVars, err := testenv.GetAWSEnv(pod, deployment.GetName()) Expect(err).To(Succeed(), "Failed to get AWS env variables from Ingestor Cluster pod") - testenv.ValidateContent(envVars, awsEnvVars, true) + Expect(testenv.ValidateContent(envVars, awsEnvVars, true)).To(Succeed(), "AWS env variable validation failed") } if strings.Contains(pod, "ingest") { // Verify default-mode.conf testcaseEnvInst.Log.Info("Verify default-mode.conf") - testenv.ValidateContent(defaultsConf, defaultsIngest, true) + Expect(testenv.ValidateContent(defaultsConf, defaultsIngest, true)).To(Succeed(), "default-mode.conf validation failed") } else if strings.Contains(pod, "idxc") { // Verify inputs.conf - testcaseEnvInst.Log.Info("Verify inputs.conf") - inputsPath := "opt/splunk/etc/system/local/inputs.conf" - inputsConf, err := testenv.GetConfFile(pod, inputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get inputs.conf from Indexer Cluster pod") - testenv.ValidateContent(inputsConf, inputs, true) + Expect(testenv.VerifyConfFileContent(pod, "opt/splunk/etc/system/local/inputs.conf", deployment.GetName(), inputs, "Failed to get inputs.conf from Indexer Cluster pod")).To(Succeed(), "inputs.conf verification failed") } } }) diff --git a/test/ingest_search/ingest_search_suite_test.go b/test/ingest_search/ingest_search_suite_test.go index ab8b3992c..1f5c72724 100644 --- a/test/ingest_search/ingest_search_suite_test.go +++ b/test/ingest_search/ingest_search_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,11 +11,10 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package ingestsearchtest +package ingestsearch import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,23 +22,13 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "ingest-" + testenv.RandomDNSName(3) ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - +// TestIngestSearch is the main entry point +func TestIngestSearch(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -51,11 +40,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/ingest_search/ingest_search_test.go b/test/ingest_search/ingest_search_test.go index 531fecbb3..3767a9688 100644 --- a/test/ingest_search/ingest_search_test.go +++ b/test/ingest_search/ingest_search_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,254 +11,37 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package ingestsearchtest +package ingestsearch import ( - "bufio" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("Ingest and Search Test", func() { +var _ = Describe("Ingest and Search test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var firstLine string BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can search internal logs for standalone instance", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - searchString := "index=_internal | stats count by host" - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) - - var searchResults map[string]interface{} - unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - if unmarshalErr != nil { - testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") - } - - prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - searchString := "index=_internal GUID component=ServerConfig" - - // Perform a simple search - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString, deployment) - if reqErr != nil { - testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - if statusErr != nil { - testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - if resErr != nil { - testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - - // Display results for debug purposes - prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + RunS1InternalLogSearchTest(ctx, deployment, testcaseEnvInst) }) - }) - Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can ingest custom data to new index and search", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify splunk status is up - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - splunkBin := "/opt/splunk/bin/splunk" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - splunkCmd := "status" - - statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) - command := []string{"/bin/bash"} - statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) - return enterpriseApi.PhaseError - } - - if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { - testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) - return enterpriseApi.PhaseError - } - - testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - // Create an index - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - indexName := "myTestIndex" - - // Create an index on a standalone instance - err = testenv.CreateAnIndexStandalone(ctx, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed response to add index to splunk") - - // Create a mock logfile to ingest - logFile := "/tmp/test.log" - err = testenv.CreateMockLogfile(logFile, 1) - Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) - - // Copy log file and ingest it - err = testenv.IngestFileViaOneshot(ctx, logFile, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) - - // Read first line to find a search token - var file, openErr = os.Open(logFile) - Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) - - reader := bufio.NewReader(file) - var readErr error - firstLine, readErr = reader.ReadString('\n') - Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) - - tokens := strings.Fields(firstLine) - Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) - - searchToken := tokens[len(tokens)-1] - testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) - - searchString := fmt.Sprintf("index=%s | stats count by host", indexName) - - // Wait for search results to be available instead of fixed sleep - err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for search results") - - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) - - // Verify result. Should get count 1. result:{count:1} - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) - - hostCount := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) - testHostCnt := strings.Compare(hostCount["count"].(string), "1") - testHostname := strings.Compare(hostCount["host"].(string), podName) - Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) - - searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString2, deployment) - Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) - - testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) - var searchResults2 testenv.SearchJobResultsResponse - jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - found := false - for key, elem := range searchResults2.Results { - testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) - trimFirstLine := strings.TrimSuffix(firstLine, "\n") - if strings.Compare(elem.Raw, trimFirstLine) == 0 { - testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) - found = true - } - } - Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) + RunS1IngestAndSearchTest(ctx, deployment, testcaseEnvInst) }) }) }) diff --git a/test/ingest_search/ingest_search_test_shared.go b/test/ingest_search/ingest_search_test_shared.go new file mode 100644 index 000000000..a44d62764 --- /dev/null +++ b/test/ingest_search/ingest_search_test_shared.go @@ -0,0 +1,173 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ingestsearch + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" + + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// RunS1InternalLogSearchTest deploys a Standalone instance and verifies internal log searches +// using both synchronous and asynchronous search APIs. +func RunS1InternalLogSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + _, err := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, "", "") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") + + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + + // Verify sync search on _internal index + syncSearchString := "index=_internal | stats count by host" + Eventually(func() error { + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, podName, syncSearchString) + if err != nil { + return fmt.Errorf("failed to execute sync search: %w", err) + } + + var searchResults map[string]interface{} + if err := json.Unmarshal([]byte(searchResultsResp), &searchResults); err != nil { + return fmt.Errorf("failed to unmarshal JSON response: %w", err) + } + + prettyResults, err := json.MarshalIndent(searchResults, "", " ") + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to generate pretty JSON") + } else { + testcaseEnvInst.Log.Info("Sync search results", "prettyResults", string(prettyResults)) + } + + return nil + }, deployment.GetTimeout(), testenv.PollInterval).Should(Succeed(), "Sync search on _internal index failed") + + // Verify async search on _internal index + asyncSearchString := "index=_internal GUID component=ServerConfig" + Eventually(func() error { + sid, err := testenv.PerformSearchReq(ctx, deployment, podName, asyncSearchString) + if err != nil { + return fmt.Errorf("failed to execute async search: %w", err) + } + testcaseEnvInst.Log.Info("Got a search with SID", "sid", sid) + + searchStatusResult, err := testenv.GetSearchStatus(ctx, deployment, podName, sid) + if err != nil { + return fmt.Errorf("failed to get search status: %w", err) + } + testcaseEnvInst.Log.Info("Search status", "searchStatusResult", searchStatusResult) + + searchResultsResp, err := testenv.GetSearchResults(ctx, deployment, podName, sid) + if err != nil { + return fmt.Errorf("failed to get search results: %w", err) + } + + prettyResults, err := json.MarshalIndent(searchResultsResp, "", " ") + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to generate pretty JSON") + } else { + testcaseEnvInst.Log.Info("Async search results", "prettyResults", string(prettyResults)) + } + + return nil + }, deployment.GetTimeout(), testenv.PollInterval).Should(Succeed(), "Async search on _internal index failed") +} + +// RunS1IngestAndSearchTest deploys a Standalone instance, ingests a custom log file into a new +// index, and verifies the ingested data is searchable via both sync and async search APIs. +func RunS1IngestAndSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + _, err := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, "", "") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") + + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + indexName := "myTestIndex" + + err = testenv.CreateAnIndexStandalone(ctx, deployment, indexName, podName) + Expect(err).To(Succeed(), "Failed to add index to Standalone") + + logFile := "/tmp/test.log" + err = testenv.CreateMockLogfile(logFile, 1) + Expect(err).To(Succeed(), "Failed to create mock logfile %s", logFile) + + err = testenv.IngestFileViaOneshot(ctx, deployment, logFile, indexName, podName) + Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) + + file, openErr := os.Open(logFile) + Expect(openErr).To(Succeed(), "Failed to open logfile %s", logFile) + defer file.Close() + + reader := bufio.NewReader(file) + firstLine, readErr := reader.ReadString('\n') + Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod %s", logFile, podName) + + tokens := strings.Fields(firstLine) + Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) + + searchToken := tokens[len(tokens)-1] + testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) + + searchString := fmt.Sprintf("index=%s | stats count by host", indexName) + + err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) + Expect(err).To(Succeed(), "Timed out waiting for search results") + + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, podName, searchString) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) + + var searchResults map[string]interface{} + jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON search results from response '%s'", searchResultsResp) + + testcaseEnvInst.Log.Info("Search results", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) + + hostCount := searchResults["result"].(map[string]interface{}) + testcaseEnvInst.Log.Info("Sync search results host count", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) + Expect(hostCount["count"].(string)).To(Equal("1"), "Incorrect search results for count. Expected: 1 Got: %s", hostCount["count"].(string)) + Expect(hostCount["host"].(string)).To(Equal(podName), "Incorrect search result hostname. Expected: %s Got: %s", podName, hostCount["host"].(string)) + + searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) + sid, reqErr := testenv.PerformSearchReq(ctx, deployment, podName, searchString2) + Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString2, podName) + testcaseEnvInst.Log.Info("Got a search with SID", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, deployment, podName, sid) + Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for SID %s", podName, sid) + testcaseEnvInst.Log.Info("Search status", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, deployment, podName, sid) + Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for SID %s", podName, sid) + + testcaseEnvInst.Log.Info("Raw search results", "searchResultsResp", searchResultsResp) + var searchResults2 testenv.SearchJobResultsResponse + jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON search results from response '%s'", searchResultsResp) + + trimFirstLine := strings.TrimSuffix(firstLine, "\n") + found := false + for key, elem := range searchResults2.Results { + testcaseEnvInst.Log.Info("Search results _raw and host", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) + if elem.Raw == trimFirstLine { + testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) + found = true + } + } + Expect(found).To(BeTrue(), "Incorrect search results %s", searchResults) +} diff --git a/test/licensemanager/manager_suite_test.go b/test/licensemanager/licensemanager_suite_test.go similarity index 69% rename from test/licensemanager/manager_suite_test.go rename to test/licensemanager/licensemanager_suite_test.go index f06d487e8..4e55572a1 100644 --- a/test/licensemanager/manager_suite_test.go +++ b/test/licensemanager/licensemanager_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package licensemanager import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,22 +22,13 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "lmanager-" + testenv.RandomDNSName(3) ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { +// TestLicenseManager is the main entry point +func TestLicenseManager(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -50,11 +40,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/licensemanager/lm_s1_test.go b/test/licensemanager/lm_s1_test.go deleted file mode 100644 index 1512d906e..000000000 --- a/test/licensemanager/lm_s1_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with License Manager", func() { - It("licensemanager, smoke, s1: Splunk Operator can configure License Manager with Standalone in S1 SVA", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - RunLMS1Test(ctx, deployment, testcaseEnvInst, config) - }) - }) -}) diff --git a/test/licensemanager/lm_test.go b/test/licensemanager/lm_test.go new file mode 100644 index 000000000..d8080473b --- /dev/null +++ b/test/licensemanager/lm_test.go @@ -0,0 +1,105 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package licensemanager + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// masterManagerLMConfigs defines the V3 (master) and V4 (manager) variants +// for the license manager tests. +var masterManagerLMConfigs = []testenv.MasterManagerLMTestConfig{ + {NamePrefix: "master", Label: "licensemaster", NewConfig: testenv.NewLicenseMasterConfig}, + {NamePrefix: "", Label: "licensemanager", NewConfig: testenv.NewLicenseManagerConfig}, +} + +var _ = Describe("License Manager test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + for _, tc := range masterManagerLMConfigs { + tc := tc + + Context("Standalone deployment (S1) with "+tc.Label, func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", smoke, s1: Splunk Operator can configure LM with Standalone in S1 SVA", func() { + RunLMS1Test(ctx, deployment, testcaseEnvInst, tc.NewConfig()) + }) + }) + + Context("Clustered deployment (C3) with "+tc.Label, func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, c3: Splunk Operator can configure LM with Indexers and Search Heads in C3 SVA", func() { + RunLMC3Test(ctx, deployment, testcaseEnvInst, tc.NewConfig()) + }) + }) + + Context("Multisite cluster deployment (M4) with "+tc.Label, func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, m4: Splunk Operator can configure LM with Indexers and Search Heads in M4 SVA", func() { + RunLMM4Test(ctx, deployment, testcaseEnvInst, tc.NewConfig()) + }) + }) + + Context("Clustered deployment (C3) with "+tc.Label+" App Framework", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { + RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, tc.NewConfig()) + }) + }) + } +}) diff --git a/test/licensemanager/lm_test_shared.go b/test/licensemanager/lm_test_shared.go index 205fd2c74..9418696fc 100644 --- a/test/licensemanager/lm_test_shared.go +++ b/test/licensemanager/lm_test_shared.go @@ -20,238 +20,64 @@ import ( "path/filepath" "time" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + . "github.com/onsi/gomega" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" - - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" ) -type LicenseTestConfig struct { - DeployStandaloneWithLM func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) - LicenseManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - ClusterManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - DeployMultisiteClusterWithSearchHead func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas, siteCount int, mcRef string) error - DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) - LicenseManagerPodName string - LicenseManagerSpecType string -} - -func NewLicenseMasterConfig() *LicenseTestConfig { - return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterMasterReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, - DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { - return deployment.DeployLicenseMasterWithGivenSpec(ctx, name, spec.(enterpriseApiV3.LicenseMasterSpec)) - }, - LicenseManagerPodName: testenv.LicenseMasterPod, - LicenseManagerSpecType: "v3", - } -} - -func NewLicenseManagerConfig() *LicenseTestConfig { - return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLM(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterManagerReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, - DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { - return deployment.DeployLicenseManagerWithGivenSpec(ctx, name, spec.(enterpriseApi.LicenseManagerSpec)) - }, - LicenseManagerPodName: testenv.LicenseManagerPod, - LicenseManagerSpecType: "v4", - } -} - -func downloadLicenseAndCreateConfigMap(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv) { - downloadDir := "licenseFolder" - - var err error - var licenseFilePath string - - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err = testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - case "azure": - licenseFilePath, err = testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - case "gcp": - licenseFilePath, err = testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) -} - -func downloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) { - switch testenv.ClusterProvider { - case "eks": - err := testenv.DownloadFilesFromS3(testDataS3Bucket, appDir, downloadDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) - case "azure": - containerName := "/" + azureDataContainer + "/" + appDir - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDir, containerName, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) - case "gcp": - err := testenv.DownloadFilesFromGCP(testDataS3Bucket, appDir, downloadDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) - } -} - -func uploadAppFiles(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, testS3Bucket, testDir, downloadDir string, appFileList []string, version string) []string { - var uploadedFiles []string - var err error - - switch testenv.ClusterProvider { - case "eks": - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", version)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, testDir, appFileList, downloadDir) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3", version)) - case "azure": - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", version)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDir, testDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure", version)) - case "gcp": - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP", version)) - uploadedFiles, err = testenv.UploadFilesToGCP(testS3Bucket, testDir, appFileList, downloadDir) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP", version)) - } - - return uploadedFiles -} - -func deleteUploadedFiles(ctx context.Context, testS3Bucket string, uploadedApps []string) { - switch testenv.ClusterProvider { - case "eks": - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - case "azure": - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - case "gcp": - testenv.DeleteFilesOnGCP(testS3Bucket, uploadedApps) - } -} +// RunLMS1Test deploys a Standalone with License Manager and Monitoring Console, +// then verifies LM is configured on the standalone and MC pods. +func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.LicenseTestConfig) { + // Set up license config map + Expect(testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst)).To(Succeed(), "Unable to setup license config map") -func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) - - // Create standalone Deployment with License Manager/Master + // Create Standalone deployment with License Manager/Master mcRef := deployment.GetName() standalone, err := config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with LM") - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + // Wait for License Manager/Master and Standalone to be in READY status + Expect(testenv.VerifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config.ClusterReadinessConfig, standalone)).To(Succeed(), "License Manager or Standalone not ready") - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) + // Deploy and verify Monitoring Console + _, err = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Verify livenessProbe and readinessProbe config object and scripts + Expect(testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, false)).To(Succeed(), "Probe config verification failed") - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify License Manager/Master is configured on standalone instance + // Verify License Manager/Master is configured on Standalone instance standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, standalonePodName) + Expect(testenv.VerifyLMConfiguredOnPod(ctx, deployment, standalonePodName)).To(Succeed(), "LM not configured on Standalone pod") // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + Expect(testenv.VerifyLMConfiguredOnMC(ctx, deployment)).To(Succeed(), "LM not configured on MC") } -func RunLMC3Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) - - // Deploy Single site Cluster with License Manager/Master +// RunLMC3Test deploys a C3 cluster with License Manager and Monitoring Console, +// then verifies LM is configured on indexers, search heads, and MC. +func RunLMC3Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.LicenseTestConfig) { + // Deploy single site Cluster with License Manager/Master mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + Expect(config.DeployC3WithLicense(ctx, deployment, testcaseEnvInst, 3, true, mcRef)).To(Succeed(), "Unable to deploy C3 with license") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testenv.DeployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef)).To(Succeed(), "Unable to deploy MC and verify RFSF") - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + // Verify License Manager/Master is configured on indexers, search heads, and MC + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), 3, false, 0) + Expect(testenv.VerifyLMConfiguredOnCluster(ctx, deployment, indexerPods)).To(Succeed(), "LM not configured on cluster pods") } -func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, testenvInstance *testenv.TestEnv, config *LicenseTestConfig) { +// RunLMC3AppFrameworkTest deploys a License Manager with App Framework, verifies V1 apps +// are installed, upgrades to V2 apps, and verifies the updated apps. +func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, testenvInstance *testenv.TestEnv, config *testenv.LicenseTestConfig) { var ( appListV1 []string appListV2 []string testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") testDataS3Bucket = os.Getenv("TEST_BUCKET") - AzureDataContainer = os.Getenv("TEST_CONTAINER") + azureDataContainer = os.Getenv("TEST_CONTAINER") appDirV1 = testenv.AppLocationV1 appDirV2 = testenv.AppLocationV2 currDir, _ = os.Getwd() @@ -267,112 +93,57 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment appFileList := testenv.GetAppFileList(appListV1) // Download V1 Apps - downloadAppFiles(ctx, testDataS3Bucket, AzureDataContainer, appDirV1, downloadDirV1, appFileList, appVersion) + Expect(testenv.DownloadAppFiles(ctx, testDataS3Bucket, azureDataContainer, appDirV1, downloadDirV1, appFileList, appVersion)).To(Succeed(), "Unable to download V1 app files") // Upload V1 apps testDir = "lm-" + testenv.RandomDNSName(4) - uploadedFiles := uploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV1, appFileList, appVersion) + uploadedFiles, err := testenv.UploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV1, appFileList, appVersion) + Expect(err).To(Succeed(), "Unable to upload V1 app files") uploadedApps = append(uploadedApps, uploadedFiles...) - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Set up license config map + Expect(testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst)).To(Succeed(), "Unable to setup license config map") - // Create App framework Spec + // Create app framework spec volumeName := "lm-test-volume-" + testenv.RandomDNSName(3) - var volumeSpec []enterpriseApi.VolumeSpec - switch testenv.ClusterProvider { - case "eks": - volumeSpec = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - case "azure": - volumeSpec = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(volumeName, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - case "gcp": - volumeSpec = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "gcp", "blob", testenv.GetDefaultS3Region())} - } + volumeSpec := testcaseEnvInst.GenerateVolumeSpecForProvider(ctx, volumeName) - // AppSourceDefaultSpec: Remote Storage volume name and Scope of App deployment + // AppSourceDefaultSpec: Remote Storage volume name and scope of app deployment appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: volumeName, Scope: enterpriseApi.ScopeLocal, } - // appSourceSpec: App source name, location and volume name and scope from appSourceDefaultSpec + // appSourceSpec: app source name, location and volume name and scope from appSourceDefaultSpec appSourceName := "lm-" + testenv.RandomDNSName(3) appSourceSpec := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceName, testDir, appSourceDefaultSpec)} - // appFrameworkSpec: AppSource settings, Poll Interval, volumes, appSources on volumes + // appFrameworkSpec: AppSource settings, poll interval, volumes, appSources on volumes appFrameworkSpec := enterpriseApi.AppFrameworkSpec{ Defaults: appSourceDefaultSpec, AppsRepoPollInterval: 60, VolList: volumeSpec, AppSources: appSourceSpec, } - - var spec interface{} - if config.LicenseManagerSpecType == "v3" { - spec = enterpriseApiV3.LicenseMasterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } else { - spec = enterpriseApi.LicenseManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } + spec := config.BuildLMAppFrameworkSpec(testcaseEnvInst, appFrameworkSpec) // Deploy the License Manager/Master with App Framework - var err error _, err = config.DeployLicenseManagerWithGivenSpec(ctx, deployment, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy LM with App framework") + Expect(err).To(Succeed(), "Unable to deploy LM with App Framework") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + Expect(config.LicenseManagerReady(ctx, deployment, testcaseEnvInst)).To(Succeed(), "License Manager not ready") - // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) - podName := []string{fmt.Sprintf(config.LicenseManagerPodName, deployment.GetName(), 0)} - testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appListV1, true, enterpriseApi.ScopeLocal) + // Wait for V1 apps to reach Install phase on License Manager/Master + err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, deployment.GetName(), config.CrKind, appSourceName, appListV1, enterpriseApi.PhaseInstall, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for V1 apps to reach Install phase on LicenseManager") - // Verify apps are installed on License Manager/Master - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appListV1, false, "enabled", false, false) + // Verify apps are copied and installed on License Manager/Master + podName := []string{fmt.Sprintf(config.LicenseManagerPodName, deployment.GetName(), 0)} + Expect(testenv.VerifyLMAppsOnPod(ctx, deployment, testcaseEnvInst, testenvInstance, podName, appListV1, false)).To(Succeed(), "V1 apps not found on LM pod") // Delete files uploaded - deleteUploadedFiles(ctx, testS3Bucket, uploadedApps) + testenv.DeleteUploadedFiles(ctx, testS3Bucket, uploadedApps) uploadedApps = nil // Create a list of apps to upload to S3 after poll period @@ -381,81 +152,44 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment appVersion = "V2" // Download V2 Apps - downloadAppFiles(ctx, testDataS3Bucket, AzureDataContainer, appDirV2, downloadDirV2, appFileList, appVersion) + Expect(testenv.DownloadAppFiles(ctx, testDataS3Bucket, azureDataContainer, appDirV2, downloadDirV2, appFileList, appVersion)).To(Succeed(), "Unable to download V2 app files") // Upload V2 apps - uploadedFiles = uploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV2, appFileList, appVersion) + uploadedFiles, err = testenv.UploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV2, appFileList, appVersion) + Expect(err).To(Succeed(), "Unable to upload V2 app files") uploadedApps = append(uploadedApps, uploadedFiles...) - err = testcaseEnvInst.WaitForLicenseManagerPhase(ctx, deployment, testcaseEnvInst.GetName(), deployment.GetName(), enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for LicenseManager to reach Ready phase") + // Wait for operator to detect V2 apps (any app leaves Install phase) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), config.CrKind, appSourceName, appFileList) - // Verify LM stays in ready state - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + // Wait for License Manager/Master to finish processing V2 apps and become Ready + Expect(config.LicenseManagerReady(ctx, deployment, testcaseEnvInst)).To(Succeed(), "License Manager not ready after V2 upload") - // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) - testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appListV2, true, enterpriseApi.ScopeLocal) + // Wait for V2 apps to reach Install phase on License Manager/Master + err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, deployment.GetName(), config.CrKind, appSourceName, appListV2, enterpriseApi.PhaseInstall, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for V2 apps to reach Install phase on LicenseManager") - // Verify apps are installed on License Manager/Master - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appListV2, true, "enabled", true, false) + // Verify apps are copied and installed on License Manager/Master + Expect(testenv.VerifyLMAppsOnPod(ctx, deployment, testcaseEnvInst, testenvInstance, podName, appListV2, true)).To(Succeed(), "V2 apps not found on LM pod") // Delete files uploaded - deleteUploadedFiles(ctx, testS3Bucket, uploadedApps) + testenv.DeleteUploadedFiles(ctx, testS3Bucket, uploadedApps) // Delete locally downloaded app files - os.RemoveAll(downloadDirV1) - os.RemoveAll(downloadDirV2) + Expect(os.RemoveAll(downloadDirV1)).To(Succeed(), "Unable to delete locally downloaded V1 app files") + Expect(os.RemoveAll(downloadDirV2)).To(Succeed(), "Unable to delete locally downloaded V2 app files") } -func RunLMM4Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) - +// RunLMM4Test deploys a multisite cluster with License Manager and Monitoring Console, +// then verifies LM is configured on indexers, search heads, and MC. +func RunLMM4Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.LicenseTestConfig) { // Deploy Multisite Cluster with License Manager/Master and Search Head siteCount := 3 mcRef := deployment.GetName() - err := config.DeployMultisiteClusterWithSearchHead(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Multisite Indexer Cluster status - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(config.DeployM4WithLicense(ctx, deployment, testcaseEnvInst, 1, siteCount, mcRef)).To(Succeed(), "Unable to deploy M4 with license") + Expect(testenv.DeployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef)).To(Succeed(), "Unable to deploy MC and verify RFSF") - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 2, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 3, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + // Verify License Manager/Master is configured on indexers, search heads, and MC + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) + Expect(testenv.VerifyLMConfiguredOnCluster(ctx, deployment, indexerPods)).To(Succeed(), "LM not configured on multisite cluster pods") } diff --git a/test/licensemanager/manager_lm_c3_test.go b/test/licensemanager/manager_lm_c3_test.go deleted file mode 100644 index a4507295d..000000000 --- a/test/licensemanager/manager_lm_c3_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure License Manager with Indexers and Search Heads in C3 SVA", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemanager/manager_lm_m4_test.go b/test/licensemanager/manager_lm_m4_test.go deleted file mode 100644 index d896147c6..000000000 --- a/test/licensemanager/manager_lm_m4_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Manager", func() { - It("licensemanager, integration, m4: Splunk Operator can configure License Manager with indexers and search head in M4 SVA", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { - RunLMM4Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - -}) diff --git a/test/licensemaster/lm_c3_test.go b/test/licensemaster/lm_c3_test.go deleted file mode 100644 index 821e7c65e..000000000 --- a/test/licensemaster/lm_c3_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - licensemanager.RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - licensemanager.RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemaster/lm_m4_test.go b/test/licensemaster/lm_m4_test.go deleted file mode 100644 index 1e3ec8190..000000000 --- a/test/licensemaster/lm_m4_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Master", func() { - It("licensemaster, integration, m4: Splunk Operator can configure License Master with indexers and search head in M4 SVA", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - licensemanager.RunLMM4Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - -}) diff --git a/test/licensemaster/lm_s1_test.go b/test/licensemaster/lm_s1_test.go deleted file mode 100644 index 066bc3c4c..000000000 --- a/test/licensemaster/lm_s1_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with License Master", func() { - It("licensemaster, smoke, s1: Splunk Operator can configure License Master with Standalone in S1 SVA", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - licensemanager.RunLMS1Test(ctx, deployment, testcaseEnvInst, config) - }) - }) -}) diff --git a/test/licensemaster/lm_suite_test.go b/test/licensemaster/lm_suite_test.go deleted file mode 100644 index c1ac356c2..000000000 --- a/test/licensemaster/lm_suite_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "lm-" + testenv.RandomDNSName(3) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - - sc, _ := GinkgoConfiguration() - sc.Timeout = testenv.MediumSuiteTimeout - - RunSpecs(t, "Running "+testSuiteName, sc) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } -}) diff --git a/test/monitoring_console/manager_monitoring_console_test.go b/test/monitoring_console/manager_monitoring_console_test.go deleted file mode 100644 index ef14ec9b7..000000000 --- a/test/monitoring_console/manager_monitoring_console_test.go +++ /dev/null @@ -1,1099 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package monitoringconsoletest - -import ( - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Monitoring Console test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Deploy Monitoring Console", func() { - It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - /* - Test Steps - 1. Deploy Monitoring Console - 2. Deploy Standalone - 3. Wait for Monitoring Console status to go back to READY - 4. Verify Standalone configured in Monitoring Console Config Map - 5. Verify Monitoring Console Pod has correct peers in Peer List - --------------- RECONFIG WITH NEW MC -------------------------- - 6. Reconfig S1 with 2nd Monitoring Console Name - 7. Check 2nd Monitoring Console Config Map to verify s1 - 8. Deploy 2nd Monitoring Console Pod - 9. Verify Standalone pod is configured on Monitoring Console Pod - 10. Verify 1st Monitoring Console Config Map is not configured with S1 - 11. Verify 1st Monitoring Console Pod is not configured with S1 - */ - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Create Standalone Spec and apply - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### - - // Reconfig S1 with 2nd Monitoring Console Name - mcTwoName := deployment.GetName() + "-two" - err = deployment.GetInstance(ctx, standaloneOneName, standaloneOne) - Expect(err).To(Succeed(), "Unable to get instance of Standalone") - standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName - - // Update Standalone with 2nd MC - err = deployment.UpdateCR(ctx, standaloneOne) - Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") - - // Deploy 2nd MC Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Second Monitoring Console Pod") - - // Verify 2nd Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on SECOND MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcTwoName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in SECOND MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcTwoName, true, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is not configured in MC ONE Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod NOT ON FIRST MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod Not in MC ONE Peer List - testcaseEnvInst.Log.Info("Check standalone NOT ON FIRST MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - // CSPL-901 standaloneTwoName := deployment.GetName() + "-two" - standaloneTwoName := "standalone-" + testenv.RandomDNSName(3) - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete Standlone TWO of the standalone and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment with Scale up", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Scale Standalone to 2 REPLICAS - 8. Wait for Second Standalone POD to come up and PHASE to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone PODS configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 1, false, 0) - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Scale Standalone instance - testcaseEnvInst.Log.Info("Scaling Standalone CR") - scaledReplicaCount := 2 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale Standalone") - - // Ensure standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings - 6. Verify Monitoring Console Pod has peers(indexers) in Peer string - 7. Scale SH Cluster - 8. Scale Indexer Count - 9. Add a standalone - 10. Verify Standalone is configured in MC Config Map and Peer String - 11. Verify SH are configured in MC Config Map and Peers String - 12. Verify Indexers are configured in Peer String - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Cluster Manager to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager in MC ConfigMap") - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // Scale Search Head Cluster - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Scale indexers - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready Phase - // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) - - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc1, integration: MC can configure SHC, indexer instances and reconfigure to new MC", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ################# Update Monitoring Console In Cluster Manager CR ################################## - - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Manager Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // ################# Update Monitoring Console In SHC CR ################################## - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") - - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify MC is Ready and stays in ready state - // testenv.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo, testcaseEnvInst) - - // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) - - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC two config after SHC reconfig") - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for indexers in MC two config after SHC reconfig") - - // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) - - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads to be removed from MC one config after SHC reconfig") - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - }) - }) - - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("managermc2, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - /* - Test Steps - 1. Deploy Multisite Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Manager to use 2nd Monitoring Console - 9. Verify Cluster Manager is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Manager not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod - */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Manager RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - /* - Test Steps - 1. Deploy Standalone with name "search-head-adhoc" - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone with name "search-head" - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := "search-head-adhoc" - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standaloneOne to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneOneName, standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneOne is configured in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneOne Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - standaloneTwoName := "search-head" - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource(CR) resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneTwo is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneTwo Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete standaloneTwo and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - -}) diff --git a/test/monitoring_console/monitoring_console_suite_test.go b/test/monitoring_console/monitoring_console_suite_test.go index 5ee315f8a..97dd6cbef 100644 --- a/test/monitoring_console/monitoring_console_suite_test.go +++ b/test/monitoring_console/monitoring_console_suite_test.go @@ -1,23 +1,20 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - -package monitoringconsoletest +package monitoringconsole import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,23 +22,13 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "mc-" + testenv.RandomDNSName(3) ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - +// TestMonitoringConsole is the main entry point +func TestMonitoringConsole(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -53,11 +40,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/monitoring_console/monitoring_console_test.go b/test/monitoring_console/monitoring_console_test.go index 07f69c70a..41e81a773 100644 --- a/test/monitoring_console/monitoring_console_test.go +++ b/test/monitoring_console/monitoring_console_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,574 +11,282 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package monitoringconsoletest +package monitoringconsole import ( - "fmt" + "context" "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) -var _ = Describe("Monitoring Console test", func() { +// masterManagerMCConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 reconfig and M4 MC reconfig test tables. +var masterManagerMCConfigs = []testenv.MCVersionConfig{ + { + MCReconfigParams: testenv.MCReconfigParams{CMServiceNameFmt: testenv.ClusterMasterServiceName, CMURLKey: "SPLUNK_CLUSTER_MASTER_URL"}, + NamePrefix: "master", + Label: "mastermc", + DeployC3WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, shc bool, mcRef string) error { + return d.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, name, replicas, shc, mcRef) + }, + DeployM4WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error { + return d.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, name, replicas, siteCount, mcRef, shc) + }, + NewCMObject: func() client.Object { return &enterpriseApiV3.ClusterMaster{} }, + VerifyCMReady: func(ctx context.Context, d *testenv.Deployment, te *testenv.TestCaseEnv) error { + return te.VerifyClusterMasterReady(ctx, d) + }, + SHCReconfigTimeout: 0, + VerifyMCTwoReadyAfterSHC: true, + }, + { + MCReconfigParams: testenv.MCReconfigParams{CMServiceNameFmt: testenv.ClusterManagerServiceName, CMURLKey: splcommon.ClusterManagerURL}, + NamePrefix: "", + Label: "managermc", + DeployC3WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, shc bool, mcRef string) error { + return d.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, name, replicas, shc, mcRef) + }, + DeployM4WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error { + return d.DeployMultisiteClusterWithMonitoringConsole(ctx, name, replicas, siteCount, mcRef, shc) + }, + NewCMObject: func() client.Object { return &enterpriseApi.ClusterManager{} }, + VerifyCMReady: func(ctx context.Context, d *testenv.Deployment, te *testenv.TestCaseEnv) error { + return te.VerifyClusterManagerReady(ctx, d) + }, + SHCReconfigTimeout: 5 * time.Minute, + VerifyMCTwoReadyAfterSHC: false, + }, +} + +// C3 scale-up tests — V3 (master) and V4 (manager) variants +var _ = Describe("Monitoring Console C3 scale-up tests", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + + for _, cfg := range masterManagerMCConfigs { + cfg := cfg + Context("Clustered deployment C3 scale-up ("+cfg.Label+")", func() { + BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, cfg.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(cfg.Label+", smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { + RunC3MCScaleUpTest(ctx, deployment, testcaseEnvInst, cfg) + }) + }) + } +}) + +// Manager (V4) Monitoring Console tests +var _ = Describe("Monitoring Console test (manager)", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { + Context("Deploy Monitoring Console", func() { + It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { /* Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console + 1. Deploy Monitoring Console + 2. Deploy Standalone 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings - 6. Verify Monitoring Console Pod has peers(indexers) in Peer string - 7. Scale SH Cluster - 8. Scale Indexer Count - 9. Add a standalone - 10. Verify Standalone is configured in MC Config Map and Peer String - 11. Verify SH are configured in MC Config Map and Peers String - 12. Verify Indexers are configured in Peer String + 4. Verify Standalone configured in Monitoring Console Config Map + 5. Verify Monitoring Console Pod has correct peers in Peer List + --------------- RECONFIG WITH NEW MC -------------------------- + 6. Reconfig S1 with 2nd Monitoring Console Name + 7. Check 2nd Monitoring Console Config Map to verify s1 + 8. Deploy 2nd Monitoring Console Pod + 9. Verify Standalone pod is configured on Monitoring Console Pod + 10. Verify 1st Monitoring Console Config Map is not configured with S1 + 11. Verify 1st Monitoring Console Pod is not configured with S1 */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + // Deploy Monitoring Console CRD + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Cluster Master to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Master in MC ConfigMap") - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) + // Create Standalone Spec and apply + standaloneOneName := deployment.GetName() + mcName := deployment.GetName() + standaloneOne, err := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneOneName, mcName) + Expect(err).To(Succeed(), "Unable to deploy Standalone with MC reference") - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + // wait for custom resource resource version to change and verify MC is ready + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + // Check Standalone is configured in MC + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcName, true)).To(Succeed(), "Standalone not configured in MC") - // Scale Search Head Cluster - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" + // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + // Reconfig S1 with 2nd Monitoring Console Name + mcTwoName := deployment.GetName() + "-two" + Expect(deployment.GetInstance(ctx, standaloneOneName, standaloneOne)).To(Succeed(), "Unable to get instance of Standalone") + standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") + // Update Standalone with 2nd MC + err = deployment.UpdateCR(ctx, standaloneOne) + Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + // Deploy 2nd MC Pod + _, err = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two") - // Scale indexers - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" + // Check Standalone is configured in MC Two + testcaseEnvInst.Log.Info("Checking for Standalone on SECOND MC after Standalone RECONFIG") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcTwoName, true)).To(Succeed(), "Standalone not configured in MC Two after reconfig") - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + // Verify Monitoring Console One is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed(), "MC One not ready after reconfig") - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") + // Check Standalone is NOT configured in MC One + testcaseEnvInst.Log.Info("Checking for Standalone NOT ON FIRST MC after Standalone RECONFIG") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcName, false)).To(Succeed(), "Standalone still configured in MC One after reconfig") - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + }) + }) - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready Phase - // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) - - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + Context("Standalone deployment (S1)", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, deployment.GetName(), "standalone-"+testenv.RandomDNSName(3)) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure to new MC", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { + Context("Standalone deployment with Scale up", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { /* Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod + 1. Deploy Standalone + 2. Wait for Standalone to go to READY + 3. Deploy Monitoring Console + 4. Wait for Monitoring Console status to be READY + 5. Verify Standalone configured in Monitoring Console Config Map + 6. Verify Monitoring Console Pod has correct peers in Peer List + 7. Scale Standalone to 2 REPLICAS + 8. Wait for Second Standalone POD to come up and PHASE to be READY + 9. Wait for Monitoring Console status to go UPDATING then READY + 10. Verify both Standalone PODS configured in Monitoring Console Config Map + 11. Verify both Standalone configured in Monitoring Console Pod Peers String */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 + standaloneName := deployment.GetName() mcName := deployment.GetName() + standalone, err := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneName, mcName) + Expect(err).To(Succeed(), "Unable to deploy Standalone with MC reference") - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") + // Deploy MC and wait for MC to be READY + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Check Standalone is configured in MC + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneName, mcName, true)).To(Succeed(), "Standalone not configured in MC") // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ################# Update Monitoring Console In Cluster Master CR ################################## - - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Scale Standalone instance + testcaseEnvInst.Log.Info("Scaling Standalone CR") + scaledReplicaCount := 2 + standalone = &enterpriseApi.Standalone{} + Expect(deployment.GetInstance(ctx, deployment.GetName(), standalone)).To(Succeed(), "Failed to get instance of Standalone") - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) + standalone.Spec.Replicas = int32(scaledReplicaCount) - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Master Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + err = deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Failed to scale Standalone") - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + // Ensure standalone reaches ScalingUp phase and returns to Ready + Expect(testcaseEnvInst.VerifyStandalonePhaseAndReady(ctx, deployment, enterpriseApi.PhaseScalingUp, standalone)).To(Succeed(), "Standalone did not reach ScalingUp phase or not ready after scale up") - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + // wait for custom resource resource version to change and verify MC is ready + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready after scale up") - // ################# Update Monitoring Console In SHC CR ################################## + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") - - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ############################ VERIFICATOIN FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) - - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // ############################ VERIFICATOIN FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) - - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + // Check both Standalone pods are configured in MC after scale up + testcaseEnvInst.Log.Info("Checking for Standalone Pods on MC after scale up") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true)).To(Succeed(), "Standalone pods not found in MC after scale up") }) }) - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - /* - Test Steps - 1. Deploy Multisite Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Master to use 2nd Monitoring Console - 9. Verify Cluster Master is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Master not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod - */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with SHC") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Master RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + Context("Standalone deployment (S1)", func() { + It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, "search-head-adhoc", "search-head") + }) + }) - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) +}) - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) +// C3 reconfig and M4 tests — V3 (master) and V4 (manager) variants +var _ = Describe("Monitoring Console reconfig tests", func() { - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + // C3 reconfig tests + for _, cfg := range masterManagerMCConfigs { + cfg := cfg + Context("Clustered deployment C3 reconfig ("+cfg.Label+")", func() { + BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, cfg.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(cfg.Label+", integration: MC can configure SHC, indexer instances and reconfigure to new MC", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { + RunC3MCReconfigTest(ctx, deployment, testcaseEnvInst, cfg) + }) }) - }) + } + + // M4 reconfig tests + for _, cfg := range masterManagerMCConfigs { + cfg := cfg + Context("Multisite Clustered deployment M4 reconfig ("+cfg.Label+")", func() { + BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, cfg.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(cfg.Label+", integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { + RunM4MCReconfigTest(ctx, deployment, testcaseEnvInst, cfg) + }) + }) + } }) diff --git a/test/monitoring_console/monitoring_console_test_shared.go b/test/monitoring_console/monitoring_console_test_shared.go new file mode 100644 index 000000000..77e54cc40 --- /dev/null +++ b/test/monitoring_console/monitoring_console_test_shared.go @@ -0,0 +1,261 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package monitoringconsole + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunM4MCReconfigTest deploys an M4 multisite cluster with a Monitoring Console, +// verifies the MC configuration, then reconfigures the Cluster Manager to point +// to a second MC and verifies both MCs are updated correctly. +func RunM4MCReconfigTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg testenv.MCVersionConfig) { + defaultSHReplicas := 3 + defaultIndexerReplicas := 1 + siteCount := 3 + mcName := deployment.GetName() + + err := cfg.DeployM4WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) + Expect(err).To(Succeed(), "Unable to deploy multisite cluster") + + // Ensure cluster coordinator and all M4 components are ready + Expect(testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount, func() error { + return cfg.VerifyCMReady(ctx, deployment, testcaseEnvInst) + })).To(Succeed(), "M4 components not ready") + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Generate pod name slices for verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), defaultIndexerReplicas, true, siteCount) + + // Verify MC configuration for M4 cluster + Expect(testenv.VerifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, mcName, shPods, indexerPods)).To(Succeed(), "MC config verification failed for M4 cluster") + + // ############ CLUSTER MANAGER MC RECONFIG ################################# + mcTwoName, _, err := testenv.ReconfigCMWithNewMC(ctx, deployment, testcaseEnvInst, cfg) + Expect(err).To(Succeed(), "Unable to reconfig CM with new MC") + + Expect(testenv.VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, shPods, indexerPods, false)).To(Succeed(), "MC Two verification failed after CM reconfig") + Expect(testenv.VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcName, mc, shPods, true)).To(Succeed(), "MC One verification failed after CM reconfig") +} + +// c3MCSetupResult holds the common state produced by deployAndVerifyC3WithMC. +type c3MCSetupResult struct { + mc *enterpriseApi.MonitoringConsole + mcName string + shPods []string + indexerPods []string + shReplicas int + indexerReplicas int +} + +// deployAndVerifyC3WithMC deploys a C3 cluster with a Monitoring Console, waits +// for everything to be ready, and verifies the MC configuration. This is the +// common setup shared by RunC3MCReconfigTest and RunC3MCScaleUpTest. +func deployAndVerifyC3WithMC(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg testenv.MCVersionConfig) c3MCSetupResult { + shReplicas := 3 + indexerReplicas := 3 + mcName := deployment.GetName() + + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + err = cfg.DeployC3WithMC(ctx, deployment, deployment.GetName(), indexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, func(ctx2 context.Context, d *testenv.Deployment) error { + return cfg.VerifyCMReady(ctx2, d, testcaseEnvInst) + })).To(Succeed(), "C3 cluster not ready") + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") + + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 0) + Expect(testenv.VerifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, mcName, shPods, indexerPods)).To(Succeed(), "MC config verification failed for C3 cluster") + + return c3MCSetupResult{mc: mc, mcName: mcName, shPods: shPods, indexerPods: indexerPods, shReplicas: shReplicas, indexerReplicas: indexerReplicas} +} + +// RunC3MCReconfigTest deploys a C3 single-site cluster with a Monitoring Console, +// verifies the MC configuration, then reconfigures the Cluster Manager and SHC +// to point to a second MC and verifies both MCs are updated correctly. +func RunC3MCReconfigTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg testenv.MCVersionConfig) { + setup := deployAndVerifyC3WithMC(ctx, deployment, testcaseEnvInst, cfg) + + // Verify Monitoring Console is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), setup.mc)).To(Succeed(), "Monitoring Console not ready") + + // ################# Update Monitoring Console In Cluster Manager CR ################################## + + mcTwoName, mcTwo, err := testenv.ReconfigCMWithNewMC(ctx, deployment, testcaseEnvInst, cfg) + Expect(err).To(Succeed(), "Unable to reconfig CM with new MC") + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready after CM MC reconfig") + + // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### + Expect(testenv.VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, setup.shPods, setup.indexerPods, true)).To(Succeed(), "MC Two verification failed after CM reconfig") + + // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### + Expect(testenv.VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, setup.mcName, setup.mc, setup.shPods, false)).To(Succeed(), "MC One verification failed after CM reconfig") + + // ################# Update Monitoring Console In SHC CR ################################## + + shc := &enterpriseApi.SearchHeadCluster{} + shcName := deployment.GetName() + "-shc" + Expect(testcaseEnvInst.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, shc, shcName, mcTwoName)).To(Succeed(), "Unable to update SHC MC ref") + + // Ensure Search Head Cluster goes to Ready Phase + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready after SHC MC reconfig") + + if cfg.VerifyMCTwoReadyAfterSHC { + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo)).To(Succeed(), "MC Two not ready after SHC reconfig") + } + + // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### + Expect(testenv.VerifyMCTwoAfterSHCReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, setup.shPods, setup.indexerPods, cfg.SHCReconfigTimeout)).To(Succeed(), "MC Two verification failed after SHC reconfig") + + // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### + Expect(testenv.VerifyMCOneAfterSHCReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, setup.mcName, setup.mc, setup.shPods, cfg.SHCReconfigTimeout)).To(Succeed(), "MC One verification failed after SHC reconfig") +} + +// RunC3MCScaleUpTest deploys a C3 cluster with a Monitoring Console, verifies MC +// configuration, scales SHC and indexers, adds a standalone, and verifies the MC +// is updated correctly after scale up. Works for both V3 (master) and V4 (manager). +func RunC3MCScaleUpTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg testenv.MCVersionConfig) { + setup := deployAndVerifyC3WithMC(ctx, deployment, testcaseEnvInst, cfg) + + // Scale Search Head Cluster + scaledSHReplicas := setup.shReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", setup.shReplicas, "New Replicas", scaledSHReplicas) + Expect(testcaseEnvInst.ScaleSearchHeadCluster(ctx, deployment, scaledSHReplicas)).To(Succeed(), "Unable to scale Search Head Cluster") + + // Scale indexers + scaledIndexerReplicas := setup.indexerReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", setup.indexerReplicas, "New Replicas", scaledIndexerReplicas) + Expect(testcaseEnvInst.ScaleIndexerCluster(ctx, deployment, scaledIndexerReplicas)).To(Succeed(), "Unable to scale Indexer Cluster") + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, setup.mc) + + // Deploy Standalone with MC reference + _, err := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), setup.mcName) + Expect(err).To(Succeed(), "Unable to deploy Standalone with MC reference") + + // Ensure Indexer Cluster goes to Ready phase + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready after scale up") + + // Ensure Search Head Cluster goes to Ready Phase + // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready after scale up") + + // Wait for custom resource resource version to change and verify MC is ready + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, setup.mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready after scale up") + + // Verify Standalone configured on Monitoring Console + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, deployment.GetName(), setup.mcName, true)).To(Succeed(), "Standalone not configured in MC") + + // Verify MC configuration after scale up + testcaseEnvInst.Log.Info("Verify MC configuration after Scale Up") + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) + Expect(testenv.VerifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, setup.mcName, shPods, indexerPods)).To(Succeed(), "MC config verification failed after scale up") +} + +// RunS1StandaloneAddDeleteMCTest deploys two standalone instances with a Monitoring Console, +// verifies both are registered, then deletes the second standalone and verifies the MC +// config map and peer list are updated correctly. +func RunS1StandaloneAddDeleteMCTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, standaloneOneName, standaloneTwoName string) { + mcName := deployment.GetName() + + // Deploy Standalone one with MCRef + _, err := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneOneName, mcName) + Expect(err).To(Succeed(), "Unable to deploy Standalone with MC reference") + + // Deploy MC and wait for MC to be READY + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Check Standalone is configured in MC Config Map + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true)).To(Succeed(), "Standalone pods not found in MC config") + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Add another standalone instance in namespace + testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") + standaloneTwoSpec := testenv.NewStandaloneSpecWithMCRefAndResources( + testcaseEnvInst.GetSplunkImage(), mcName, + corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("2"), + "memory": resource.MustParse("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("0.2"), + "memory": resource.MustParse("256Mi"), + }, + }, + ) + standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone two to be in READY status + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo)).To(Succeed(), "Standalone Two not ready") + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") + + // Check both standalones are configured in MC Config Map + standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true)).To(Succeed(), "Standalone pods not found in MC config after adding second standalone") + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Delete standalone two and ensure MC is updated + testcaseEnvInst.Log.Info("Deleting second standalone deployment from namespace", "Standalone Name", standaloneTwoName) + Expect(deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo)).To(Succeed(), "Unable to get standalone instance") + err = deployment.DeleteCR(ctx, standaloneTwo) + Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready after standalone deletion") + + // Check standalone one is still configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true)).To(Succeed(), "Standalone One not found in MC config after deleting second standalone") + + // Check Standalone Two NOT configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, false)).To(Succeed(), "Standalone Two still found in MC config after deletion") +} diff --git a/test/secret/manager_secret_c3_test.go b/test/secret/manager_secret_c3_test.go deleted file mode 100644 index 6728f8fb9..000000000 --- a/test/secret/manager_secret_c3_test.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managersecret, smoke, c3: secret update on indexers and search head cluster", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(12) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_m4_test.go b/test/secret/manager_secret_m4_test.go deleted file mode 100644 index 7e5c43c12..000000000 --- a/test/secret/manager_secret_m4_test.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managersecret, integration, m4: secret update on multisite indexers and search head cluster", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(12) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_s1_test.go b/test/secret/manager_secret_s1_test.go deleted file mode 100644 index 5fce6e756..000000000 --- a/test/secret/manager_secret_s1_test.go +++ /dev/null @@ -1,354 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("managersecret, integration, s1: Secret update on a standalone instance with LM and MC", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(12) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("managersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_c3_test.go b/test/secret/secret_c3_test.go deleted file mode 100644 index 150ebc126..000000000 --- a/test/secret/secret_c3_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastersecret, smoke, c3: secret update on indexers and search head cluster", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for ClusterInitialized event to confirm cluster is fully initialized - idxcName := deployment.GetName() + "-idxc" - err = testcaseEnvInst.WaitForClusterInitialized(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterInitialized event on IndexerCluster") - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(12) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on SearchHeadCluster - shcName := deployment.GetName() + "-shc" - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on IndexerCluster - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_m4_test.go b/test/secret/secret_m4_test.go deleted file mode 100644 index 5e05185ca..000000000 --- a/test/secret/secret_m4_test.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastersecret, integration, m4: secret update on multisite indexers and search head cluster", NodeTimeout(testenv.LongTimeout), func(ctx SpecContext) { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(12) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_s1_test.go b/test/secret/secret_s1_test.go deleted file mode 100644 index 563a2f361..000000000 --- a/test/secret/secret_s1_test.go +++ /dev/null @@ -1,354 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("mastersecret, integration, s1: Secret update on a standalone instance with LM and MC", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(12) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("mastersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("mastersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_suite_test.go b/test/secret/secret_suite_test.go index e40384daf..3fe838503 100644 --- a/test/secret/secret_suite_test.go +++ b/test/secret/secret_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package secret import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,23 +22,13 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 10 * ConsistentPollInterval -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "secret-" + testenv.RandomDNSName(3) ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - +// TestSecret is the main entry point +func TestSecret(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -51,11 +40,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/secret/secret_test.go b/test/secret/secret_test.go new file mode 100644 index 000000000..fb23c0e84 --- /dev/null +++ b/test/secret/secret_test.go @@ -0,0 +1,107 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// masterManagerConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 and M4 secret test tables. +var masterManagerConfigs = []testenv.MasterManagerTestConfig{ + {NamePrefix: "master", Label: "mastersecret", NewConfig: testenv.NewClusterReadinessConfigV3}, + {NamePrefix: "", Label: "managersecret", NewConfig: testenv.NewClusterReadinessConfigV4}, +} + +var _ = Describe("Secret test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + // S1 tests — V3/V4 distinction is irrelevant for standalone secret tests (always V4) + Context("Standalone deployment (S1) with LM and MC", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It("managersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + + It("managersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteTest(ctx, deployment, testcaseEnvInst, config) + }) + + It("managersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteWithMCRefTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + + // C3 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerConfigs { + tc := tc + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", smoke, c3: secret update on Indexers and Search Head Cluster", func() { + config := tc.NewConfig() + RunC3SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // M4 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix, testenv.WithTimeout(40000)) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, m4: secret update on multisite Indexers and Search Head Cluster", func() { + config := tc.NewConfig() + RunM4SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } +}) diff --git a/test/secret/secret_test_shared.go b/test/secret/secret_test_shared.go new file mode 100644 index 000000000..0b4f3493d --- /dev/null +++ b/test/secret/secret_test_shared.go @@ -0,0 +1,132 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// RunS1SecretUpdateTest runs the standard S1 secret update test workflow +func RunS1SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + setup, err := testenv.SetupS1WithLMAndMC(ctx, deployment, testcaseEnvInst, config) + Expect(err).To(Succeed(), "Unable to setup S1 with LM and MC") + + // Update Secret Value on Secret Object + updatedSecretData, err := testenv.GenerateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, setup.NamespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to generate and apply secret update") + + Expect(testenv.VerifyS1SecretChangeApplied(ctx, deployment, testcaseEnvInst, config, setup, updatedSecretData, true)).To(Succeed(), "S1 secret change not applied") +} + +// RunS1SecretDeleteTest runs the standard S1 secret delete test workflow +func RunS1SecretDeleteTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + setup, err := testenv.SetupS1WithLMAndMC(ctx, deployment, testcaseEnvInst, config) + Expect(err).To(Succeed(), "Unable to setup S1 with LM and MC") + + // Re-fetch secret struct so we can verify its data is restored after deletion + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), setup.NamespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete Secret Object + err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), setup.NamespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + Expect(testenv.VerifyS1SecretChangeApplied(ctx, deployment, testcaseEnvInst, config, setup, secretStruct.Data, false)).To(Succeed(), "S1 secret delete not applied") +} + +// RunS1SecretDeleteWithMCRefTest runs the S1 secret delete test with MC reference workflow +func RunS1SecretDeleteWithMCRefTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Create standalone Deployment with MonitoringConsoleRef + mcRef := deployment.GetName() + standalone, err := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), mcRef) + Expect(err).To(Succeed(), "Unable to deploy Standalone with MC reference") + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete secret by passing empty Data Map + err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + // Ensure standalone reaches Updating phase and returns to Ready + Expect(testcaseEnvInst.VerifyStandalonePhaseAndReady(ctx, deployment, enterpriseApi.PhaseUpdating, standalone)).To(Succeed(), "Standalone did not reach Updating phase or not ready after secret delete") + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed(), "MC version not changed or not ready") + + Expect(testenv.VerifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretStruct.Data, false)).To(Succeed(), "Secrets not propagated after delete") +} + +// RunC3SecretUpdateTest runs the standard C3 secret update test workflow +func RunC3SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + mcRef := deployment.GetName() + Expect(config.DeployC3WithLicense(ctx, deployment, testcaseEnvInst, 3, true, mcRef)).To(Succeed(), "Unable to deploy C3 with license") + + mc, resourceVersion, updatedSecretData, err := testenv.ApplySecretUpdateAndVerifyCMUpdating(ctx, deployment, testcaseEnvInst, config) + Expect(err).To(Succeed(), "Unable to apply secret update and verify CM updating") + + Expect(testenv.VerifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config)).To(Succeed(), "LM and Cluster Manager not ready") + + // Ensure Search Head Cluster goes to Ready phase + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed(), "Search Head Cluster not ready") + + // Wait for PasswordSyncCompleted event on SearchHeadCluster + shcName := deployment.GetName() + "-shc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") + + // Ensure Indexers go to Ready phase + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed(), "Indexers not ready") + + // Wait for PasswordSyncCompleted event on IndexerCluster + idxcName := deployment.GetName() + "-idxc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") + + Expect(testenv.VerifyPostSecretChangeCluster(ctx, deployment, testcaseEnvInst, mc, resourceVersion, updatedSecretData)).To(Succeed(), "Post secret change cluster verification failed") +} + +// RunM4SecretUpdateTest runs the standard M4 secret update test workflow +func RunM4SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + siteCount := 3 + mcRef := deployment.GetName() + + Expect(config.DeployM4WithLicense(ctx, deployment, testcaseEnvInst, 1, siteCount, mcRef)).To(Succeed(), "Unable to deploy M4 with license") + + mc, resourceVersion, updatedSecretData, err := testenv.ApplySecretUpdateAndVerifyCMUpdating(ctx, deployment, testcaseEnvInst, config) + Expect(err).To(Succeed(), "Unable to apply secret update and verify CM updating") + + Expect(config.LicenseManagerReady(ctx, deployment, testcaseEnvInst)).To(Succeed(), "License Manager not ready") + Expect(testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount, func() error { + return config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + })).To(Succeed(), "M4 components not ready") + + Expect(testenv.VerifyPostSecretChangeCluster(ctx, deployment, testcaseEnvInst, mc, resourceVersion, updatedSecretData)).To(Succeed(), "Post secret change cluster verification failed") +} diff --git a/test/smartstore/manager_smartstore_test.go b/test/smartstore/manager_smartstore_test.go deleted file mode 100644 index 27706547e..000000000 --- a/test/smartstore/manager_smartstore_test.go +++ /dev/null @@ -1,280 +0,0 @@ -package smartstore - -import ( - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Smartstore test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure multiple indexes through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure indexes which use default volumes through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("managersmartstore, smoke: Can configure indexes and volumes on Multisite Indexer Cluster through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-two" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } - }) - }) -}) diff --git a/test/smartstore/smartstore_suite_test.go b/test/smartstore/smartstore_suite_test.go index 5c1687d04..37e8bc65e 100644 --- a/test/smartstore/smartstore_suite_test.go +++ b/test/smartstore/smartstore_suite_test.go @@ -1,8 +1,20 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -10,23 +22,13 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smartstore-" + testenv.RandomDNSName(3) ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - +// TestSmartStore is the main entry point +func TestSmartStore(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -38,11 +40,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/smartstore/smartstore_test.go b/test/smartstore/smartstore_test.go index 9e0f74fa5..b8ea14624 100644 --- a/test/smartstore/smartstore_test.go +++ b/test/smartstore/smartstore_test.go @@ -1,361 +1,106 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( - "fmt" "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) +// smartstoreTestConfig extends MasterManagerTestConfig with a per-variant +// timeout used by the S1 multiple-indexes test. +type smartstoreTestConfig struct { + testenv.MasterManagerTestConfig + S1IndexesTimeout time.Duration +} + +// masterManagerSmartstoreConfigs defines the V3 (master) and V4 (manager) variants +// shared by the S1 and M4 smartstore test tables. +var masterManagerSmartstoreConfigs = []smartstoreTestConfig{ + {testenv.MasterManagerTestConfig{NamePrefix: "master", Label: "mastersmartstore", NewConfig: testenv.NewClusterReadinessConfigV3}, 2 * time.Minute}, + {testenv.MasterManagerTestConfig{NamePrefix: "", Label: "managersmartstore", NewConfig: testenv.NewClusterReadinessConfigV4}, 5 * time.Minute}, +} + var _ = Describe("Smartstore test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure multiple indexes through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure indexes which use default volumes through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(ctx, deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - + for _, tc := range masterManagerSmartstoreConfigs { + tc := tc + Context("Standalone deployment (S1)", func() { + BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration: Can configure multiple indexes through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + RunS1MultipleIndexesTest(ctx, deployment, testcaseEnvInst, tc.S1IndexesTimeout) + }) + + It(tc.Label+", integration: Can configure indexes which use default volumes through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + RunS1DefaultVolumesTest(ctx, deployment, testcaseEnvInst) + }) }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("mastersmartstore, m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - oldBundleHash := testenv.GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify new bundle is pushed - testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, testcaseEnvInst.GetName(), 1, oldBundleHash) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } + Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { + BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", m4, smoke: Can configure indexes and volumes on Multisite Indexer Cluster through app", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + config := tc.NewConfig() + RunM4MultisiteSmartStoreTest(ctx, deployment, testcaseEnvInst, config) + }) }) - }) + } Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Etc storage", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Etc and pvc for Var volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - EtcVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageEtcEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Var storage", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Var and pvc for Etc volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - VarVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageVarEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Etc storage", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageEtcEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, true) + }) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Var storage", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageVarEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, false) }) }) }) diff --git a/test/smartstore/smartstore_test_shared.go b/test/smartstore/smartstore_test_shared.go new file mode 100644 index 000000000..217ae35d8 --- /dev/null +++ b/test/smartstore/smartstore_test_shared.go @@ -0,0 +1,226 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package smartstore + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" +) + +// RunS1MultipleIndexesTest runs the standard S1 multiple indexes SmartStore test workflow +func RunS1MultipleIndexesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, waitTimeout time.Duration) { + volName := "test-volume-" + testenv.RandomDNSName(3) + // Each key is unique at runtime because RandomDNSName generates distinct suffixes. + indexVolumeMap := map[string]string{ + "test-index-" + testenv.RandomDNSName(3): volName, + "test-index-" + testenv.RandomDNSName(3): volName, + } + testcaseEnvInst.Log.Info("Index secret name", "secretName", testcaseEnvInst.GetIndexSecretName()) + + var indexSpec []enterpriseApi.IndexSpec + volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + // Create index volume spec from index volume map + for index, volume := range indexVolumeMap { + indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) + } + + // Generate smartstore spec + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volumeSpec, + IndexList: indexSpec, + } + + // Deploy Standalone + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Wait for Standalone to reach Ready phase + err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, waitTimeout) + Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") + + // Verify standalone goes to ready state and stays ready + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + for indexName := range indexVolumeMap { + Expect(testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName)).To(Succeed(), "Index not found on pod") + } + + // Ingest data to the index + for indexName := range indexVolumeMap { + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Unable to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, indexName, podName)).To(Succeed(), "Unable to ingest file via monitor") + } + + // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 + for indexName := range indexVolumeMap { + Expect(testenv.RollHotToWarm(ctx, deployment, podName, indexName)).To(BeTrue(), "Unable to roll hot to warm") + Expect(testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName)).To(Succeed(), "Index not found on S3") + } +} + +// RunS1DefaultVolumesTest runs the standard S1 default volumes SmartStore test workflow +func RunS1DefaultVolumesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} + defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} + cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} + + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + Defaults: defaultSmartStoreSpec, + CacheManagerConf: cacheManagerSmartStoreSpec, + } + + // Deploy Standalone with given smartstore spec + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Verify standalone goes to ready state + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + Expect(testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName)).To(Succeed(), "Index not found on pod") + + // Check special index configs + Expect(testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"])).To(Succeed(), "Index config mismatch") + + // Ingest data to the index + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + Expect(testenv.CreateMockLogfile(logFile, 2000)).To(Succeed(), "Unable to create mock logfile") + Expect(testenv.IngestFileViaMonitor(ctx, deployment, logFile, indexName, podName)).To(Succeed(), "Unable to ingest file via monitor") + + // Roll Hot Buckets on the test index by restarting splunk + Expect(testenv.RollHotToWarm(ctx, deployment, podName, indexName)).To(BeTrue(), "Unable to roll hot to warm") + + // Check for indexes on S3 + Expect(testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName)).To(Succeed(), "Index not found on S3") + + // Verify Cachemanager Values + serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" + + // Validate MaxCacheSizeMB + Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB))).To(Succeed(), "MaxCacheSizeMB mismatch") + + // Validate EvictionPaddingSizeMB + Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB))).To(Succeed(), "EvictionPaddingSizeMB mismatch") + + // Validate MaxConcurrentDownloads + Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads))).To(Succeed(), "MaxConcurrentDownloads mismatch") + + // Validate MaxConcurrentUploads + Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads))).To(Succeed(), "MaxConcurrentUploads mismatch") + + // Validate EvictionPolicy + Expect(testcaseEnvInst.VerifyConfOnPod(ctx, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy)).To(Succeed(), "EvictionPolicy mismatch") +} + +// RunS1EphemeralStorageTest deploys a Standalone with one ephemeral storage volume configured and verifies it is ready. +// Pass etcStorage=true to set EtcVolumeStorageConfig, false to set VarVolumeStorageConfig. +func RunS1EphemeralStorageTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, storageConfig enterpriseApi.StorageClassSpec, etcStorage bool) { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + } + if etcStorage { + spec.CommonSplunkSpec.EtcVolumeStorageConfig = storageConfig + } else { + spec.CommonSplunkSpec.VarVolumeStorageConfig = storageConfig + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") + + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed(), "Standalone not ready") +} + +// RunM4MultisiteSmartStoreTest runs the standard M4 multisite SmartStore test workflow +func RunM4MultisiteSmartStoreTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + } + + siteCount := 3 + err := config.DeployMultisiteClusterWithIndexes(ctx, deployment, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + Expect(testenv.VerifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount, false)).To(Succeed(), "M4 cluster or RF/SF verification failed") + + // Use multisite workflow helper to verify index, ingest data, roll to warm, and verify on S3 + Expect(testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, siteCount, indexName)).To(Succeed(), "Multisite indexer workflow failed") + + // V3 needs explicit bundle-push verification; V4 did not have this check historically + var oldBundleHash string + if config.GetAPIVersion() == "v3" { + oldBundleHash = config.GetBundleHash(ctx, deployment) + } + + testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") + indexNameTwo := "test-index-" + testenv.RandomDNSName(3) + indexList := []string{indexName, indexNameTwo} + newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} + + // Update CR with new index based on API version + Expect(config.AppendSmartStoreIndex(ctx, deployment, newIndex)).To(Succeed(), "Unable to append SmartStore index") + + // Second-round: skip VerifyIndexerClusterMultisiteStatus (already verified in + // the first round; multisite topology is unchanged by an index addition). + Expect(testenv.VerifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount, true)).To(Succeed(), "M4 cluster or RF/SF verification failed after adding index") + + if config.GetAPIVersion() == "v3" { + // Verify new bundle is pushed to all indexers + Expect(testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, 1, oldBundleHash)).To(Succeed(), "Cluster Manager bundle push not detected") + } + + // Verify both indexes on all sites + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + for _, index := range indexList { + Expect(testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index)).To(Succeed(), "Index not found on pod") + } + } + + // Use multisite workflow helper for the new index + testcaseEnvInst.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) + Expect(testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, siteCount, indexNameTwo)).To(Succeed(), "Multisite indexer workflow failed for new index") +} diff --git a/test/smoke/cluster_manager_sites_response.go b/test/smoke/cluster_manager_sites_response.go deleted file mode 100644 index f9bcdee60..000000000 --- a/test/smoke/cluster_manager_sites_response.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package smoke - -// ClusterManagerSitesResponse is a representation of the sites managed by a Splunk cluster-manager -// Endpoint: /services/cluster/manager/sites -type ClusterManagerSitesResponse struct { - Entries []ClusterManagerSitesEntry `json:"entry"` -} - -// ClusterManagerSitesEntry represents a site of an indexer cluster with its metadata -type ClusterManagerSitesEntry struct { - Name string `json:"name"` - Content ClusterManagerSitesContent `json:"content"` -} - -// ClusterManagerSitesContent represents detailed information about a site -type ClusterManagerSitesContent struct { - Peers map[string]ClusterManagerSitesPeer `json:"peers"` -} - -// ClusterManagerSitesPeer reprensents an indexer peer member of a site -type ClusterManagerSitesPeer struct { - ServerName string `json:"server_name"` -} diff --git a/test/smoke/smoke_suite_test.go b/test/smoke/smoke_suite_test.go index 3e651529a..2a2de75da 100644 --- a/test/smoke/smoke_suite_test.go +++ b/test/smoke/smoke_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package smoke import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,23 +22,13 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smoke-" + testenv.RandomDNSName(3) ) -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - +// TestSmoke is the main entry point +func TestSmoke(t *testing.T) { RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -51,11 +40,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index 0c2da67e3..a16d7b151 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,16 +14,10 @@ package smoke import ( - "fmt" - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - corev1 "k8s.io/api/core/v1" ) var _ = Describe("Smoke test", func() { @@ -33,133 +27,47 @@ var _ = Describe("Smoke test", func() { BeforeEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(NodeTimeout(testenv.SetupTeardownTimeout), func(ctx SpecContext) { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) Context("Standalone deployment (S1)", func() { It("smoke, basic, s1: can deploy a standalone instance", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + _, err := testcaseEnvInst.RunStandaloneDeploymentWorkflow(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, basic, c3: can deploy indexers and search head cluster", NodeTimeout(testenv.MediumLongTimeout), func(ctx SpecContext) { - - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + _, err := testcaseEnvInst.RunC3DeploymentWorkflow(ctx, deployment, 3, "") + Expect(err).To(Succeed(), "Unable to deploy C3 cluster") }) }) - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { It("smoke, basic, m4: can deploy indexers and search head cluster", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + _, err := testcaseEnvInst.RunM4DeploymentWorkflow(ctx, deployment, 1, 3, "") + Expect(err).To(Succeed(), "Unable to deploy M4 cluster") }) }) - Context("Multisite cluster deployment (M1 - multisite indexer cluster)", func() { + Context("Multisite cluster deployment (M1 - Multisite Indexer Cluster)", func() { It("smoke, basic: can deploy multisite indexers cluster", NodeTimeout(testenv.MediumTimeout), func(ctx SpecContext) { - - siteCount := 3 - err := deployment.DeployMultisiteCluster(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + _, err := testcaseEnvInst.RunM1DeploymentWorkflow(ctx, deployment, 1, 3) + Expect(err).To(Succeed(), "Unable to deploy M1 cluster") }) }) Context("Standalone deployment (S1) with Service Account", func() { It("smoke, basic, s1: can deploy a standalone instance attached to a service account", NodeTimeout(testenv.ShortTimeout), func(ctx SpecContext) { - // Create Service Account serviceAccountName := "smoke-service-account" - testcaseEnvInst.CreateServiceAccount(serviceAccountName) - - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ServiceAccount: serviceAccountName, - }, - } - - // Create standalone Deployment with License Manager - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify serviceAccount is configured on Pod - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(deployment, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName) + _, err := testcaseEnvInst.RunStandaloneWithServiceAccountWorkflow(ctx, deployment, serviceAccountName) + Expect(err).To(Succeed(), "Unable to deploy standalone with service account") }) }) }) diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index ad3a4b9c3..7eac9813e 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -4,13 +4,11 @@ import ( "context" "errors" "fmt" - "os" "path/filepath" "strings" "sync" "time" - gomega "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" corev1 "k8s.io/api/core/v1" wait "k8s.io/apimachinery/pkg/util/wait" @@ -109,7 +107,7 @@ var PVTestAppsLocation = "appframework/100mb_sample_apps/" // AppStagingLocOnPod is the volume on Splunk pod where apps will be copied from operator var AppStagingLocOnPod = "/operator-staging/appframework/" -// AppDownloadVolume location on Operator pod for App framework downloads +// AppDownloadVolume location on Operator pod for App Framework downloads var AppDownloadVolume = "/opt/splunk/appframework/downloadedApps/" // GenerateAppSourceSpec return AppSourceSpec struct with given values @@ -125,10 +123,10 @@ func GenerateAppSourceSpec(appSourceName string, appSourceLocation string, appSo func GetPodAppStatus(ctx context.Context, deployment *Deployment, podName string, ns string, appname string, clusterWideInstall bool) (string, string, error) { // For clusterwide install do not check for versions on deployer and cluster-manager as the apps arent installed there if clusterWideInstall && (strings.Contains(podName, "-cluster-manager-") || strings.Contains(podName, splcommon.TestClusterManagerDashed) || strings.Contains(podName, "-deployer-")) { - logf.Log.Info("Pod skipped as install is Cluter-wide", "PodName", podName) + logf.Log.Info("Pod skipped as install is Cluter-wide", "podName", podName) return "", "", nil } - output, err := GetPodAppInstallStatus(ctx, deployment, podName, ns, appname) + output, err := GetPodAppInstallStatus(ctx, deployment, podName, appname) if err != nil { return "", "", err } @@ -154,11 +152,11 @@ func GetPodInstalledAppVersion(ctx context.Context, deployment *Deployment, podN } } filePath := fmt.Sprintf("/opt/splunk/%s/%s/default/app.conf", path, appname) - logf.Log.Info("Check app version", "App", appname, "Conf file", filePath) + logf.Log.Info("Check app version", "app", appname, "confFile", filePath) confline, err := GetConfLineFromPod(ctx, podName, filePath, ns, "version", "launcher", true) if err != nil { - logf.Log.Error(err, "Failed to get version from pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get version from pod", "podName", podName) return "", err } version := strings.TrimSpace(strings.Split(confline, "=")[1]) @@ -167,7 +165,7 @@ func GetPodInstalledAppVersion(ctx context.Context, deployment *Deployment, podN } // GetPodAppInstallStatus Get the app install status -func GetPodAppInstallStatus(ctx context.Context, deployment *Deployment, podName string, ns string, appname string) (string, error) { +func GetPodAppInstallStatus(ctx context.Context, deployment *Deployment, podName string, appname string) (string, error) { stdin := fmt.Sprintf("/opt/splunk/bin/splunk display app '%s' -auth admin:$(cat /mnt/splunk-secrets/password)", appname) command := []string{"/bin/sh"} var stdout, stderr string @@ -181,31 +179,11 @@ func GetPodAppInstallStatus(ctx context.Context, deployment *Deployment, podName return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout) return strings.TrimSuffix(stdout, "\n"), nil } -// GetPodAppbtoolStatus Get the app btool status -func GetPodAppbtoolStatus(ctx context.Context, deployment *Deployment, podName string, ns string, appname string) (string, error) { - stdin := fmt.Sprintf("/opt/splunk/bin/splunk btool %s --app=%s --debug", appname, appname) - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command, "stdin", stdin) - return "", err - } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - - if len(stdout) > 0 { - if strings.Contains(strings.Split(stdout, "\n")[0], "App is disabled") { - return "DISABLED", nil - } - return "ENABLED", nil - } - return "", err -} - // GetAppFileList Get the Versioned App file list for app Names func GetAppFileList(appList []string) []string { appFileList := make([]string, 0, len(appList)) @@ -218,165 +196,213 @@ func GetAppFileList(appList []string) []string { // GetAppframeworkManualUpdateConfigMap gets config map for given manual update configmap func GetAppframeworkManualUpdateConfigMap(ctx context.Context, deployment *Deployment, ns string) (*corev1.ConfigMap, error) { ConfigMapName := fmt.Sprintf(AppframeworkManualUpdateConfigMap, ns) - logf.Log.Info("Get config map for", "CONFIG MAP NAME", ConfigMapName) + logf.Log.Info("Get config map for", "configMapName", ConfigMapName) ConfigMap, err := GetConfigMap(ctx, deployment, ns, ConfigMapName) if err != nil { logf.Log.Error(err, "Failed to get splunk manual poll Config Map") return ConfigMap, err } - logf.Log.Info("Config Map contents", "CONFIG MAP NAME", ConfigMapName, "Data", ConfigMap.Data) + logf.Log.Info("Config Map contents", "configMapName", ConfigMapName, "data", ConfigMap.Data) return ConfigMap, err } // GetAppDeploymentInfoStandalone returns AppDeploymentInfo for given standalone, appSourceName and appName -func GetAppDeploymentInfoStandalone(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoStandalone(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { standalone := &enterpriseApi.Standalone{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, standalone) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := standalone.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Standalone AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Standalone AppInfo Struct", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfoIngestorCluster returns AppDeploymentInfo for given IngestorCluster, appSourceName and appName -func GetAppDeploymentInfoIngestorCluster(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoIngestorCluster(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { ingestor := &enterpriseApi.IngestorCluster{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, ingestor) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := ingestor.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Ingestor AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Ingestor AppInfo Struct", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfoMonitoringConsole returns AppDeploymentInfo for given Monitoring Console, appSourceName and appName -func GetAppDeploymentInfoMonitoringConsole(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoMonitoringConsole(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { mc := &enterpriseApi.MonitoringConsole{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, mc) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := mc.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Monitoring Console AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Monitoring Console AppInfo Struct", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfoClusterManager returns AppDeploymentInfo for given Cluster Manager, appSourceName and appName -func GetAppDeploymentInfoClusterManager(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoClusterManager(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { cm := &enterpriseApi.ClusterManager{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Cluster Manager AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Cluster Manager AppInfo Struct", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } -// GetAppDeploymentInfoClusterMaster returns AppDeploymentInfo for given Cluster Manager, appSourceName and appName -func GetAppDeploymentInfoClusterMaster(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +// GetAppDeploymentInfoClusterMaster returns AppDeploymentInfo for given Cluster Master, appSourceName and appName +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoClusterMaster(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { cm := &enterpriseApiV3.ClusterMaster{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Cluster Master AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Cluster Master AppInfo Struct", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfoSearchHeadCluster returns AppDeploymentInfo for given Search Head Cluster, appSourceName and appName -func GetAppDeploymentInfoSearchHeadCluster(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoSearchHeadCluster(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { cm := &enterpriseApi.SearchHeadCluster{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Search Head Cluster AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Search Head Cluster AppInfo Struct", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appDeploymentInfo", appInfo) + if strings.Contains(appName, appInfo.AppName) { + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appDeploymentInfo", appInfo) + appDeploymentInfo = appInfo + return appDeploymentInfo, nil + } + } + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appInfoList", appInfoList) + return appDeploymentInfo, err +} + +// GetAppDeploymentInfoLicenseManager returns AppDeploymentInfo for given License Manager, appSourceName and appName +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoLicenseManager(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { + lm := &enterpriseApi.LicenseManager{} + appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} + err := deployment.GetInstance(ctx, name, lm) + if err != nil { + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) + return appDeploymentInfo, err + } + appInfoList := lm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList + for _, appInfo := range appInfoList { + testenvInstance.Log.Info("Checking License Manager AppInfo Struct", "appName", appName, "appSource", appSourceName, "licenseManagerName", name, "appDeploymentInfo", appInfo) + if strings.Contains(appName, appInfo.AppName) { + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "licenseManagerName", name, "appDeploymentInfo", appInfo) + appDeploymentInfo = appInfo + return appDeploymentInfo, nil + } + } + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "licenseManagerName", name, "appInfoList", appInfoList) + return appDeploymentInfo, err +} + +// GetAppDeploymentInfoLicenseMaster returns AppDeploymentInfo for given License Master, appSourceName and appName +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoLicenseMaster(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { + lm := &enterpriseApiV3.LicenseMaster{} + appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} + err := deployment.GetInstance(ctx, name, lm) + if err != nil { + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) + return appDeploymentInfo, err + } + appInfoList := lm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList + for _, appInfo := range appInfoList { + testenvInstance.Log.Info("Checking License Master AppInfo Struct", "appName", appName, "appSource", appSourceName, "licenseMasterName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "licenseMasterName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "licenseMasterName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfo returns AppDeploymentInfo for given CR Kind, appSourceName and appName -func GetAppDeploymentInfo(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfo(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { var appDeploymentInfo enterpriseApi.AppDeploymentInfo var err error switch crKind { case "Standalone": - appDeploymentInfo, err = GetAppDeploymentInfoStandalone(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoStandalone(ctx, deployment, name, appSourceName, appName) case "IngestorCluster": - appDeploymentInfo, err = GetAppDeploymentInfoIngestorCluster(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoIngestorCluster(ctx, deployment, name, appSourceName, appName) case "MonitoringConsole": - appDeploymentInfo, err = GetAppDeploymentInfoMonitoringConsole(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoMonitoringConsole(ctx, deployment, name, appSourceName, appName) case "SearchHeadCluster": - appDeploymentInfo, err = GetAppDeploymentInfoSearchHeadCluster(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoSearchHeadCluster(ctx, deployment, name, appSourceName, appName) case "ClusterManager": - appDeploymentInfo, err = GetAppDeploymentInfoClusterManager(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoClusterManager(ctx, deployment, name, appSourceName, appName) case "ClusterMaster": - appDeploymentInfo, err = GetAppDeploymentInfoClusterMaster(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoClusterMaster(ctx, deployment, name, appSourceName, appName) + case "LicenseManager": + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoLicenseManager(ctx, deployment, name, appSourceName, appName) + case "LicenseMaster": + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoLicenseMaster(ctx, deployment, name, appSourceName, appName) default: message := fmt.Sprintf("Failed to fetch AppDeploymentInfo. Incorrect CR Kind %s", crKind) err = errors.New(message) @@ -386,26 +412,8 @@ func GetAppDeploymentInfo(ctx context.Context, deployment *Deployment, testenvIn } // GenerateAppFrameworkSpec Generate Appframework spec -func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeName string, scope string, appSourceName string, s3TestDir string, pollInterval int) enterpriseApi.AppFrameworkSpec { - var volumeSpec []enterpriseApi.VolumeSpec - - // Create App framework volume - switch ClusterProvider { - case "eks": - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3", GetDefaultS3Region())} - case "azure": - managedID := os.Getenv("AZURE_MANAGED_ID_ENABLED") - if managedID == "false" { - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzure(volumeName, GetAzureEndpoint(ctx), testenvInstance.GetIndexSecretName(), "azure", "blob")} - } else { - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob")} - } - case "gcp": - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultS3Region())} - - default: - testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") - } +func (testenvInstance *TestCaseEnv) GenerateAppFrameworkSpec(ctx context.Context, volumeName string, scope string, appSourceName string, s3TestDir string, pollInterval int) enterpriseApi.AppFrameworkSpec { + volumeSpec := testenvInstance.GenerateVolumeSpecForProvider(ctx, volumeName) // AppSourceDefaultSpec: Remote Storage volume name and Scope of App deployment appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -429,17 +437,15 @@ func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, // WaitforPhaseChange Wait for timeout or when phase change is seen on a CR for any particular app // Deprecated: Use WaitForAppPhaseChange instead for better timeout control -func WaitforPhaseChange(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string, appSourceName string, appList []string) { - if err := WaitForAppPhaseChange(ctx, deployment, testenvInstance, name, crKind, appSourceName, appList, 2*time.Minute); err != nil { - testenvInstance.Log.Error(err, "WaitforPhaseChange did not observe a phase transition within timeout", "cr", name, "kind", crKind, "appSource", appSourceName) - } +func (testenvInstance *TestCaseEnv) WaitforPhaseChange(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string) { + _ = testenvInstance.WaitForAppPhaseChange(ctx, deployment, name, crKind, appSourceName, appList, 2*time.Minute) } // WaitForAppPhaseChange waits for any app in the list to change from PhaseInstall to another phase -func WaitForAppPhaseChange(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string, appSourceName string, appList []string, timeout time.Duration) error { +func (testenvInstance *TestCaseEnv) WaitForAppPhaseChange(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string, timeout time.Duration) error { return wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { for _, appName := range appList { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenvInstance, name, crKind, appSourceName, appName) + appDeploymentInfo, err := testenvInstance.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) if err != nil { testenvInstance.Log.Error(err, "Failed to get app deployment info") continue @@ -453,7 +459,7 @@ func WaitForAppPhaseChange(ctx context.Context, deployment *Deployment, testenvI } // VerifyAppFrameworkState will perform several verifications needed between the different steps of App Framework tests -func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deployment *Deployment, appSource []AppSourceInfo, splunkPodAge map[string]string, clusterManagerBundleHash string) string { +func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deployment *Deployment, appSource []AppSourceInfo, splunkPodAge map[string]string, clusterManagerBundleHash string) (string, error) { /* Function Steps * Verify apps 'download' and 'podCopy' states for all CRs (PARALLELIZED) * Verify apps packages are deleted from the operator pod for all CRs @@ -484,7 +490,9 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym verifyCtx, cancel := context.WithTimeout(ctx, deployment.GetTimeout()) defer cancel() - testenv.VerifyAppListPhase(verifyCtx, deployment, as.CrName, as.CrKind, as.CrAppSourceName, p, as.CrAppFileList) + if err := testenv.VerifyAppListPhase(verifyCtx, deployment, as.CrName, as.CrKind, as.CrAppSourceName, p, as.CrAppFileList); err != nil { + errChan <- fmt.Errorf("phase %v verification failed for CR %s/%s: %w", p, as.CrKind, as.CrName, err) + } }(appSourceItem, phase) } } @@ -492,20 +500,26 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym wg.Wait() close(errChan) + var phaseErrs []error for err := range errChan { if err != nil { testenv.Log.Error(err, "Parallel phase verification failed") - gomega.Expect(err).To(gomega.Succeed()) + phaseErrs = append(phaseErrs, err) } } + if len(phaseErrs) > 0 { + return "", fmt.Errorf("parallel download/podCopy phase verification failed: %w", errors.Join(phaseErrs...)) + } testenv.Log.Info("Parallel verification of app download and podCopy phases completed successfully") // Verify apps packages are deleted from the operator pod for all CRs - opPod := GetOperatorPodName(testenv) + opPod := testenv.GetOperatorPodName() for _, appSource := range appSource { testenv.Log.Info(fmt.Sprintf("Verify apps %s packages are deleted from the operator pod for CR %v with name %v", appSource.CrAppVersion, appSource.CrKind, appSource.CrName)) opPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenv.GetName(), appSource.CrKind, deployment.GetName(), appSource.CrAppScope, appSource.CrAppSourceName) - testenv.VerifyAppsPackageDeletedOnOperatorContainer(ctx, deployment, testenv.GetName(), []string{opPod}, appSource.CrAppFileList, opPath) + if err := testenv.VerifyAppsPackageDeletedOnOperatorContainer(ctx, deployment, []string{opPod}, appSource.CrAppFileList, opPath); err != nil { + return "", fmt.Errorf("apps packages not deleted from operator pod for CR %s/%s: %w", appSource.CrKind, appSource.CrName, err) + } } // Verify apps 'install' state for all CRs IN PARALLEL @@ -527,19 +541,25 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym verifyCtx, cancel := context.WithTimeout(ctx, deployment.GetTimeout()) defer cancel() - testenv.VerifyAppListPhase(verifyCtx, deployment, as.CrName, as.CrKind, as.CrAppSourceName, enterpriseApi.PhaseInstall, as.CrAppFileList) + if err := testenv.VerifyAppListPhase(verifyCtx, deployment, as.CrName, as.CrKind, as.CrAppSourceName, enterpriseApi.PhaseInstall, as.CrAppFileList); err != nil { + errChan <- fmt.Errorf("install phase verification failed for CR %s/%s: %w", as.CrKind, as.CrName, err) + } }(appSourceItem) } wg.Wait() close(errChan) + var installErrs []error for err := range errChan { if err != nil { testenv.Log.Error(err, "Parallel install phase verification failed") - gomega.Expect(err).To(gomega.Succeed()) + installErrs = append(installErrs, err) } } + if len(installErrs) > 0 { + return "", fmt.Errorf("parallel install phase verification failed: %w", errors.Join(installErrs...)) + } testenv.Log.Info("Parallel verification of app install phase completed successfully") // Verify apps packages are deleted from the CR pods @@ -547,21 +567,27 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym podDownloadPath := AppStagingLocOnPod + appSource.CrAppSourceVolumeName pod := appSource.CrPod testenv.Log.Info(fmt.Sprintf("Verify %s apps packages are deleted on pod %s", appSource.CrAppVersion, pod)) - testenv.VerifyAppsPackageDeletedOnContainer(ctx, deployment, testenv.GetName(), pod, appSource.CrAppFileList, podDownloadPath) + if err := testenv.VerifyAppsPackageDeletedOnContainer(ctx, deployment, pod, appSource.CrAppFileList, podDownloadPath); err != nil { + return "", fmt.Errorf("apps packages not deleted on pod %s for CR %s/%s: %w", pod, appSource.CrKind, appSource.CrName, err) + } } // Verify bundle push status for _, appSource := range appSource { if (appSource.CrKind == "ClusterManager" || appSource.CrKind == "ClusterMaster") && appSource.CrAppScope == enterpriseApi.ScopeCluster { testenv.Log.Info(fmt.Sprintf("Verify Cluster Manager bundle push status (%s apps) and compare bundle hash with previous bundle hash", appSource.CrAppVersion)) - testenv.VerifyClusterManagerBundlePush(ctx, deployment, testenv.GetName(), appSource.CrReplicas, clusterManagerBundleHash) + if err := testenv.VerifyClusterManagerBundlePush(ctx, deployment, appSource.CrReplicas, clusterManagerBundleHash); err != nil { + return "", fmt.Errorf("cluster manager bundle push verification failed: %w", err) + } if clusterManagerBundleHash == "" { clusterManagerBundleHash = GetClusterManagerBundleHash(ctx, deployment, appSource.CrKind) } } if appSource.CrKind == "SearchHeadCluster" && appSource.CrAppScope == enterpriseApi.ScopeCluster { testenv.Log.Info(fmt.Sprintf("Verify Deployer bundle push status (%s apps)", appSource.CrAppVersion)) - testenv.VerifyDeployerBundlePush(ctx, deployment, testenv.GetName(), appSource.CrReplicas) + if err := testenv.VerifyDeployerBundlePush(ctx, deployment, testenv.GetName(), appSource.CrReplicas); err != nil { + return "", fmt.Errorf("deployer bundle push verification failed: %w", err) + } } } @@ -569,12 +595,18 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym for _, appSource := range appSource { if appSource.CrAppScope == enterpriseApi.ScopeLocal { testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'local' scope are copied to /etc/apps/ for CR %s with name %s", appSource.CrAppVersion, appSource.CrKind, appSource.CrName)) - testenv.VerifyAppsCopied(ctx, deployment, testenv.GetName(), appSource.CrPod, appSource.CrAppList, true, appSource.CrAppScope) + if err := testenv.VerifyAppsCopied(ctx, deployment, appSource.CrPod, appSource.CrAppList, true, appSource.CrAppScope); err != nil { + return "", fmt.Errorf("local apps not copied for CR %s/%s: %w", appSource.CrKind, appSource.CrName, err) + } } else { testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'cluster' scope are NOT copied to /etc/apps/ on %v pod", appSource.CrAppVersion, appSource.CrPod)) - testenv.VerifyAppsCopied(ctx, deployment, testenv.GetName(), appSource.CrPod, appSource.CrAppList, false, appSource.CrAppScope) + if err := testenv.VerifyAppsCopied(ctx, deployment, appSource.CrPod, appSource.CrAppList, false, appSource.CrAppScope); err != nil { + return "", fmt.Errorf("cluster apps unexpectedly copied to CR pod for %s/%s: %w", appSource.CrKind, appSource.CrName, err) + } testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'cluster' scope are copied on %v pods", appSource.CrAppVersion, appSource.CrClusterPods)) - testenv.VerifyAppsCopied(ctx, deployment, testenv.GetName(), appSource.CrClusterPods, appSource.CrAppList, true, appSource.CrAppScope) + if err := testenv.VerifyAppsCopied(ctx, deployment, appSource.CrClusterPods, appSource.CrAppList, true, appSource.CrAppScope); err != nil { + return "", fmt.Errorf("cluster apps not copied to cluster pods for %s/%s: %w", appSource.CrKind, appSource.CrName, err) + } } } @@ -584,18 +616,22 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym checkUpdated := appSource.CrAppVersion == "V2" if appSource.CrAppScope == "local" { testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'local' scope for CR %s with name %s are installed on pod %s", appSource.CrAppVersion, appSource.CrKind, appSource.CrName, allPodNames)) - testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, false) + if err := testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, false); err != nil { + return "", fmt.Errorf("local apps not installed for CR %s/%s: %w", appSource.CrKind, appSource.CrName, err) + } } else { allPodNames = appSource.CrClusterPods testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'cluster' scope for CR %s with name %s are installed on pods %s", appSource.CrAppVersion, appSource.CrKind, appSource.CrName, allPodNames)) - testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, true) + if err := testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, true); err != nil { + return "", fmt.Errorf("cluster apps not installed for CR %s/%s: %w", appSource.CrKind, appSource.CrName, err) + } } } - return clusterManagerBundleHash + return clusterManagerBundleHash, nil } // GetIsDeploymentInProgressFlag returns IsDeploymentInProgress for given CR Name, CR Kind -func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string) (bool, error) { +func (testenvInstance *TestCaseEnv) GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, name string, crKind string) (bool, error) { var isDeploymentInProgress bool var err error switch crKind { @@ -603,7 +639,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.Standalone{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -611,7 +647,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.MonitoringConsole{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -619,7 +655,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.SearchHeadCluster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -627,7 +663,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApiV3.ClusterMaster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -635,7 +671,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.ClusterManager{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -643,7 +679,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApiV3.LicenseMaster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -651,7 +687,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.LicenseManager{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress diff --git a/test/testenv/assertion_helpers.go b/test/testenv/assertion_helpers.go new file mode 100644 index 000000000..10641c0f5 --- /dev/null +++ b/test/testenv/assertion_helpers.go @@ -0,0 +1,208 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ScaleSearchHeadCluster scales a Search Head Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleSearchHeadCluster(ctx context.Context, deployment *Deployment, newReplicas int) error { + shcName := deployment.GetName() + "-shc" + + // Get instance of current SHC CR with latest config + shc := &enterpriseApi.SearchHeadCluster{} + if err := deployment.GetInstance(ctx, shcName, shc); err != nil { + return fmt.Errorf("failed to get instance of Search Head Cluster: %w", err) + } + + // Update Replicas of SHC + shc.Spec.Replicas = int32(newReplicas) + if err := deployment.UpdateCR(ctx, shc); err != nil { + return fmt.Errorf("failed to scale Search Head Cluster: %w", err) + } + + // Verify Search Head Cluster scales up and goes to ScalingUp phase + return testcaseenv.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) +} + +// ScaleIndexerCluster scales an Indexer Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleIndexerCluster(ctx context.Context, deployment *Deployment, newReplicas int) error { + idxcName := deployment.GetName() + "-idxc" + + // Get instance of current Indexer CR with latest config + idxc := &enterpriseApi.IndexerCluster{} + if err := deployment.GetInstance(ctx, idxcName, idxc); err != nil { + return fmt.Errorf("failed to get instance of Indexer Cluster: %w", err) + } + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(newReplicas) + if err := deployment.UpdateCR(ctx, idxc); err != nil { + return fmt.Errorf("failed to scale Indexer Cluster: %w", err) + } + + // Verify Indexer Cluster scales up and goes to ScalingUp phase + return testcaseenv.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) +} + +// UpdateMonitoringConsoleRefAndVerify updates the MonitoringConsoleRef in a CR and waits for the change to apply +func (testcaseenv *TestCaseEnv) UpdateMonitoringConsoleRefAndVerify(ctx context.Context, deployment *Deployment, obj client.Object, instanceName string, newMCName string) error { + // Get current resource version before update + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, obj) + + // Update the MonitoringConsoleRef based on the type + switch cr := obj.(type) { + case *enterpriseApi.ClusterManager: + if err := deployment.GetInstance(ctx, instanceName, cr); err != nil { + return fmt.Errorf("failed to get instance %s: %w", instanceName, err) + } + cr.Spec.MonitoringConsoleRef.Name = newMCName + if err := deployment.UpdateCR(ctx, cr); err != nil { + return fmt.Errorf("failed to update MonitoringConsoleRef: %w", err) + } + case *enterpriseApiV3.ClusterMaster: + if err := deployment.GetInstance(ctx, instanceName, cr); err != nil { + return fmt.Errorf("failed to get instance %s: %w", instanceName, err) + } + cr.Spec.MonitoringConsoleRef.Name = newMCName + if err := deployment.UpdateCR(ctx, cr); err != nil { + return fmt.Errorf("failed to update MonitoringConsoleRef: %w", err) + } + case *enterpriseApi.SearchHeadCluster: + if err := deployment.GetInstance(ctx, instanceName, cr); err != nil { + return fmt.Errorf("failed to get instance %s: %w", instanceName, err) + } + cr.Spec.MonitoringConsoleRef.Name = newMCName + if err := deployment.UpdateCR(ctx, cr); err != nil { + return fmt.Errorf("failed to update MonitoringConsoleRef: %w", err) + } + } + + // Wait for custom resource version to change + return testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, obj, resourceVersion) +} + +// DeployStandaloneWithMCRef deploys a standalone instance with a MonitoringConsoleRef +func (testcaseenv *TestCaseEnv) DeployStandaloneWithMCRef(ctx context.Context, deployment *Deployment, deploymentName string, mcName string) (*enterpriseApi.Standalone, error) { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseenv.GetSplunkImage(), + }, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deploymentName, spec) + if err != nil { + return nil, fmt.Errorf("unable to deploy standalone instance: %w", err) + } + + // Wait for Standalone to be in READY status + if err = testcaseenv.VerifyStandaloneReady(ctx, deployment, deploymentName, standalone); err != nil { + return nil, fmt.Errorf("standalone not ready: %w", err) + } + + return standalone, nil +} + +// VerifyStandaloneInMC verifies that a standalone instance is configured in the MC +func (testcaseenv *TestCaseEnv) VerifyStandaloneInMC(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shouldExist bool) error { + standalonePod := fmt.Sprintf(StandalonePod, deploymentName, 0) + if err := testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{standalonePod}, "SPLUNK_STANDALONE_URL", mcName, shouldExist); err != nil { + return err + } + return testcaseenv.VerifyPodsInMCConfigString(ctx, []string{standalonePod}, mcName, shouldExist, false) +} + +// VerifyLMConfiguredOnPods verifies License Manager is configured on all given pods +func VerifyLMConfiguredOnPods(ctx context.Context, deployment *Deployment, podNames []string) error { + for _, podName := range podNames { + if err := VerifyLMConfiguredOnPod(ctx, deployment, podName); err != nil { + return err + } + } + return nil +} + +// VerifyM1ClusterReady verifies the cluster coordinator, indexers, and multisite status are ready (no SHC). +func (testcaseenv *TestCaseEnv) VerifyM1ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment) error) error { + if err := verifyCoordinator(ctx, deployment); err != nil { + return err + } + if err := testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + return testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) +} + +// VerifyM4ClusterReady verifies the cluster coordinator, indexers, multisite status, and SHC are ready. +// It delegates to VerifyM4ComponentsReady, adapting the callback signature. +func (testcaseenv *TestCaseEnv) VerifyM4ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment) error) error { + return testcaseenv.VerifyM4ComponentsReady(ctx, deployment, siteCount, func() error { + return verifyCoordinator(ctx, deployment) + }) +} + +// VerifyM4IndexersAndSHCReady verifies the cluster coordinator, indexers, and SHC are ready (without multisite check). +func (testcaseenv *TestCaseEnv) VerifyM4IndexersAndSHCReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment) error) error { + if err := verifyCoordinator(ctx, deployment); err != nil { + return err + } + if err := testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + return testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyC3ClusterReady verifies the cluster coordinator, SHC, and single-site indexers are ready. +func (testcaseenv *TestCaseEnv) VerifyC3ClusterReady(ctx context.Context, deployment *Deployment, verifyCoordinator func(context.Context, *Deployment) error) error { + if err := verifyCoordinator(ctx, deployment); err != nil { + return err + } + if err := testcaseenv.VerifySearchHeadClusterReady(ctx, deployment); err != nil { + return err + } + return testcaseenv.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// IngestDataOnIndexers ingests test data on all indexer pods +func IngestDataOnIndexers(ctx context.Context, deployment *Deployment, indexerCount int) { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deployment.GetName(), i) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, deployment, logFile, DefaultIngestIndex, podName) + } +} + +// IngestDataOnMultisiteIndexers ingests test data on all multisite indexer pods +func IngestDataOnMultisiteIndexers(ctx context.Context, deployment *Deployment, siteCount int) { + for site := 1; site <= siteCount; site++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), site, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, deployment, logFile, DefaultIngestIndex, podName) + } +} diff --git a/test/testenv/azureutils.go b/test/testenv/azureutils.go index 84a3f0601..28c2897ca 100644 --- a/test/testenv/azureutils.go +++ b/test/testenv/azureutils.go @@ -338,7 +338,7 @@ func (client *AzureBlobClient) DownloadFileFromAzure(ctx context.Context, downlo return "", err } - logf.Log.Info("Download from Azure successful:", "File", downloadRequest.RemoteFile) + logf.Log.Info("Download from Azure successful", "file", downloadRequest.RemoteFile) return localFile.Name(), err } @@ -354,7 +354,7 @@ func DownloadFilesFromAzure(ctx context.Context, endPoint, accountKey, accountNa } _, err := azureBlobClient.DownloadFileFromAzure(ctx, downloadRequest, endPoint, StorageAccountKey, StorageAccount) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -374,7 +374,7 @@ func DownloadLicenseFromAzure(ctx context.Context, downloadDir string) (string, azureBlobClient := &AzureBlobClient{} filename, err := azureBlobClient.DownloadFileFromAzure(ctx, downloadRequest, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount) if err != nil { - logf.Log.Error(err, "Unable to download license file", "File", filename) + logf.Log.Error(err, "Unable to download license file", "file", filename) } return filename, err } @@ -431,7 +431,7 @@ func UploadFilesToAzure(ctx context.Context, accountName, accountKey, uploadFrom fileFullPath := "https://" + StorageAccount + ".blob.core.windows.net" + "/" + azureIndexesContainer + "/" + containerName + "/" + key fileName, err := UploadFileToAzure(ctx, accountName, accountKey, fileFullPath, fileLocation) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } uploadedFiles = append(uploadedFiles, fileName) diff --git a/test/testenv/cmutil.go b/test/testenv/cmutil.go index a1b71f3ce..cf2f5300b 100644 --- a/test/testenv/cmutil.go +++ b/test/testenv/cmutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,7 +33,7 @@ type ClusterManagerSitesResponse struct { Entries []ClusterManagerSitesEntry `json:"entry"` } -// ClusterManagerSitesEntry represents a site of an indexer cluster with its metadata +// ClusterManagerSitesEntry represents a site of an Indexer Cluster with its metadata type ClusterManagerSitesEntry struct { Name string `json:"name"` Content ClusterManagerSitesContent `json:"content"` @@ -44,7 +44,7 @@ type ClusterManagerSitesContent struct { Peers map[string]ClusterManagerSitesPeer `json:"peers"` } -// ClusterManagerSitesPeer reprensents an indexer peer member of a site +// ClusterManagerSitesPeer represents an indexer peer member of a site type ClusterManagerSitesPeer struct { ServerName string `json:"server_name"` } @@ -55,7 +55,7 @@ type ClusterManagerHealthResponse struct { Entries []ClusterManagerHealthEntry `json:"entry"` } -// ClusterManagerHealthEntry represents a site of an indexer cluster with its metadata +// ClusterManagerHealthEntry represents a site of an Indexer Cluster with its metadata type ClusterManagerHealthEntry struct { Name string `json:"name"` Content ClusterManagerHealthContent `json:"content"` @@ -73,13 +73,18 @@ type ClusterManagerHealthContent struct { SiteSearchFactorMet string `json:"site_search_factor_met"` } -// CheckRFSF check if cluster has met replication factor and search factor -func CheckRFSF(ctx context.Context, deployment *Deployment) bool { - //code to execute - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") +// GetCMPodName returns the cluster-manager or cluster-master pod name based on +// whether the deployment name contains "master". +func GetCMPodName(deployment *Deployment) string { if strings.Contains(deployment.GetName(), "master") { - podName = fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-master") + return fmt.Sprintf(ClusterMasterPod, deployment.GetName()) } + return fmt.Sprintf(ClusterManagerPod, deployment.GetName()) +} + +// CheckRFSF check if cluster has met replication factor and search factor +func CheckRFSF(ctx context.Context, deployment *Deployment) bool { + podName := GetCMPodName(deployment) stdin := "curl -ks -u admin:$(cat /mnt/splunk-secrets/password) https://localhost:8089/services/cluster/manager/health?output_mode=json" command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) @@ -124,7 +129,7 @@ type ClusterManagerPeersAndSearchHeadResponse struct { } `json:"entry"` } -// GetIndexersOrSearchHeadsOnCM get indexers or search head on Cluster Manager +// GetIndexersOrSearchHeadsOnCM gets Indexers or Search Heads on Cluster Manager func GetIndexersOrSearchHeadsOnCM(ctx context.Context, deployment *Deployment, endpoint string) ClusterManagerPeersAndSearchHeadResponse { url := "" if endpoint == "sh" { @@ -132,11 +137,7 @@ func GetIndexersOrSearchHeadsOnCM(ctx context.Context, deployment *Deployment, e } else { url = "https://localhost:8089/services/cluster/manager/peers?output_mode=json" } - //code to execute - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - if strings.Contains(deployment.name, "master") { - podName = fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-master") - } + podName := GetCMPodName(deployment) stdin := fmt.Sprintf("curl -ks -u admin:$(cat /mnt/splunk-secrets/password) %s", url) command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) @@ -153,12 +154,12 @@ func GetIndexersOrSearchHeadsOnCM(ctx context.Context, deployment *Deployment, e return restResponse } -// CheckIndexerOnCM check given Indexer on cluster manager +// CheckIndexerOnCM checks given Indexer on Cluster Manager func CheckIndexerOnCM(ctx context.Context, deployment *Deployment, indexerName string) bool { restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "peer") found := false for _, entry := range restResponse.Entry { - logf.Log.Info("Peer found On CM", "Indexer Name", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Peer found On CM", "indexerName", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Label == indexerName { found = true break @@ -167,12 +168,12 @@ func CheckIndexerOnCM(ctx context.Context, deployment *Deployment, indexerName s return found } -// CheckSearchHeadOnCM check given search head on cluster manager +// CheckSearchHeadOnCM checks given Search Head on Cluster Manager func CheckSearchHeadOnCM(ctx context.Context, deployment *Deployment, searchHeadName string) bool { restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh") found := false for _, entry := range restResponse.Entry { - logf.Log.Info("Search Head On CM", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Search Head On CM", "searchHead", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Label == searchHeadName { found = true break @@ -181,12 +182,12 @@ func CheckSearchHeadOnCM(ctx context.Context, deployment *Deployment, searchHead return found } -// CheckSearchHeadRemoved check if search head is removed from Indexer Cluster +// CheckSearchHeadRemoved checks if Search Head is removed from Indexer Cluster func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh") searchHeadRemoved := true for _, entry := range restResponse.Entry { - logf.Log.Info("Search Found", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Search Found", "searchHead", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Status == "Disconnected" { searchHeadRemoved = false } @@ -194,21 +195,6 @@ func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { return searchHeadRemoved } -// RollHotBuckets roll hot buckets in cluster -func RollHotBuckets(ctx context.Context, deployment *Deployment) bool { - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - stdin := "/opt/splunk/bin/splunk rolling-restart cluster-peers -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - - return strings.Contains(stdout, "Rolling restart of all cluster peers has been initiated.") -} - // ClusterManagerInfoEndpointResponse is represtentation of /services/cluster/manager/info endpoint type ClusterManagerInfoEndpointResponse struct { Entry []struct { @@ -244,21 +230,8 @@ func ClusterManagerInfoResponse(ctx context.Context, deployment *Deployment, pod // CheckRollingRestartStatus checks if rolling restart is happening in cluster func CheckRollingRestartStatus(ctx context.Context, deployment *Deployment) bool { - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - stdin := "curl -ks -u admin:$(cat /mnt/splunk-secrets/password) https://localhost:8089/services/cluster/manager/info?output_mode=json" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - restResponse := ClusterManagerInfoEndpointResponse{} - err = json.Unmarshal([]byte(stdout), &restResponse) - if err != nil { - logf.Log.Error(err, "Failed to parse cluster searchheads") - return false - } + podName := GetCMPodName(deployment) + restResponse := ClusterManagerInfoResponse(ctx, deployment, podName) rollingRestart := true for _, entry := range restResponse.Entry { rollingRestart = entry.Content.RollingRestartFlag @@ -275,7 +248,7 @@ func CMBundlePushstatus(ctx context.Context, deployment *Deployment, previousBun // Check if new bundle was pushed by comparing hash if previousBundleHash != "" { if entry.Content.BundleID == previousBundleHash { - logf.Log.Info("Bundle hash not updated", "old Bundle hash", previousBundleHash, "new Bundle hash", entry.Content.BundleID) + logf.Log.Info("Bundle hash not updated", "oldBundleHash", previousBundleHash, "newBundleHash", entry.Content.BundleID) continue } } @@ -298,6 +271,6 @@ func GetClusterManagerBundleHash(ctx context.Context, deployment *Deployment, cr restResponse := ClusterManagerInfoResponse(ctx, deployment, podName) bundleHash := restResponse.Entry[0].Content.ActiveBundle.Checksum - logf.Log.Info("Bundle Hash on Cluster Manager Found", "Hash", bundleHash) + logf.Log.Info("Bundle Hash on Cluster Manager Found", "hash", bundleHash) return bundleHash } diff --git a/test/testenv/common_test_patterns.go b/test/testenv/common_test_patterns.go new file mode 100644 index 000000000..d08867959 --- /dev/null +++ b/test/testenv/common_test_patterns.go @@ -0,0 +1,385 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/splunk/enterprise" + corev1 "k8s.io/api/core/v1" +) + +// ClusterCoordinator abstracts the v3/v4 API differences for cluster +// manager and license manager operations. +type ClusterCoordinator interface { + LicenseManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error + ClusterManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error + DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) + DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error + DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error + VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error + DeleteClusterManager(ctx context.Context, deployment *Deployment) error + AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) error + GetBundleHash(ctx context.Context, deployment *Deployment) string + ClusterManagerPVCType() string + GetAPIVersion() string +} + +// clusterMasterCoordinator implements ClusterCoordinator for v3 (ClusterMaster/LicenseMaster). +type clusterMasterCoordinator struct{} + +func (c *clusterMasterCoordinator) LicenseManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) +} + +func (c *clusterMasterCoordinator) ClusterManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyClusterMasterReady(ctx, deployment) +} + +func (c *clusterMasterCoordinator) DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) { + return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) +} + +func (c *clusterMasterCoordinator) DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error { + return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) +} + +func (c *clusterMasterCoordinator) DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { + return deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) +} + +func (c *clusterMasterCoordinator) VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) +} + +func (c *clusterMasterCoordinator) DeleteClusterManager(ctx context.Context, deployment *Deployment) error { + return GetAndDeleteCR(ctx, deployment, &enterpriseApiV3.ClusterMaster{}, deployment.GetName()) +} + +func (c *clusterMasterCoordinator) AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) error { + name := deployment.GetName() + cm := &enterpriseApiV3.ClusterMaster{} + if err := deployment.GetInstance(ctx, name, cm); err != nil { + return fmt.Errorf("failed to get instance of Cluster Master: %w", err) + } + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + if err := deployment.UpdateCR(ctx, cm); err != nil { + return fmt.Errorf("failed to add new index to Cluster Master: %w", err) + } + return nil +} + +func (c *clusterMasterCoordinator) GetBundleHash(ctx context.Context, deployment *Deployment) string { + return GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") +} + +func (c *clusterMasterCoordinator) ClusterManagerPVCType() string { + return "cluster-master" +} + +func (c *clusterMasterCoordinator) GetAPIVersion() string { + return "v3" +} + +// clusterManagerCoordinator implements ClusterCoordinator for v4 (ClusterManager/LicenseManager). +type clusterManagerCoordinator struct{} + +func (c *clusterManagerCoordinator) LicenseManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) +} + +func (c *clusterManagerCoordinator) ClusterManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyClusterManagerReady(ctx, deployment) +} + +func (c *clusterManagerCoordinator) DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) { + return deployment.DeployStandaloneWithLM(ctx, name, mcRef) +} + +func (c *clusterManagerCoordinator) DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error { + return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) +} + +func (c *clusterManagerCoordinator) DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { + return deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) +} + +func (c *clusterManagerCoordinator) VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) +} + +func (c *clusterManagerCoordinator) DeleteClusterManager(ctx context.Context, deployment *Deployment) error { + return GetAndDeleteCR(ctx, deployment, &enterpriseApi.ClusterManager{}, deployment.GetName()) +} + +func (c *clusterManagerCoordinator) AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) error { + name := deployment.GetName() + cm := &enterpriseApi.ClusterManager{} + if err := deployment.GetInstance(ctx, name, cm); err != nil { + return fmt.Errorf("failed to get instance of Cluster Manager: %w", err) + } + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + if err := deployment.UpdateCR(ctx, cm); err != nil { + return fmt.Errorf("failed to add new index to Cluster Manager: %w", err) + } + return nil +} + +func (c *clusterManagerCoordinator) GetBundleHash(ctx context.Context, deployment *Deployment) string { + return GetClusterManagerBundleHash(ctx, deployment, "ClusterManager") +} + +func (c *clusterManagerCoordinator) ClusterManagerPVCType() string { + return "cluster-manager" +} + +func (c *clusterManagerCoordinator) GetAPIVersion() string { + return "v4" +} + +// ClusterReadinessConfig embeds a ClusterCoordinator and provides composed +// deployment and verification workflows shared across test packages. +type ClusterReadinessConfig struct { + ClusterCoordinator +} + +// MasterManagerTestConfig pairs a name prefix and test label with a factory +// function that returns the appropriate ClusterReadinessConfig. +// This is the standard config type shared by test packages that loop over +// V3 (master) and V4 (manager) variants. +// See also MasterManagerLMTestConfig (in lmutil.go) for the license-manager +// equivalent that returns *LicenseTestConfig instead. +type MasterManagerTestConfig struct { + NamePrefix string + Label string + NewConfig func() *ClusterReadinessConfig +} + +// NewClusterReadinessConfigV3 creates a ClusterReadinessConfig for v3 API (LicenseMaster/ClusterMaster) +func NewClusterReadinessConfigV3() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ClusterCoordinator: &clusterMasterCoordinator{}} +} + +// NewClusterReadinessConfigV4 creates a ClusterReadinessConfig for v4 API (LicenseManager/ClusterManager) +func NewClusterReadinessConfigV4() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ClusterCoordinator: &clusterManagerCoordinator{}} +} + +// VerifyC3ClusterReady verifies the C3 cluster is ready using the config's ClusterManagerReady callback. +func (c *ClusterReadinessConfig) VerifyC3ClusterReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyC3ClusterReady(ctx, deployment, func(ctx context.Context, d *Deployment) error { + return c.ClusterManagerReady(ctx, d, testcaseEnv) + }) +} + +// DeployMCAndGetVersion deploys and verifies a Monitoring Console, then returns both the MC +// instance and its current resource version. +func (testcaseenv *TestCaseEnv) DeployMCAndGetVersion(ctx context.Context, deployment *Deployment, name string, lmRef string) (*enterpriseApi.MonitoringConsole, string, error) { + mc, err := testcaseenv.DeployAndVerifyMonitoringConsole(ctx, deployment, name, lmRef) + if err != nil { + return nil, "", err + } + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, mc) + return mc, resourceVersion, nil +} + +// DeployAndVerifyStandalone deploys a standalone instance and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyStandalone(ctx context.Context, deployment *Deployment, mcRef string, licenseManagerRef string) (*enterpriseApi.Standalone, error) { + name := deployment.GetName() + standalone, err := deployment.DeployStandalone(ctx, name, mcRef, licenseManagerRef) + if err != nil { + return nil, fmt.Errorf("unable to deploy Standalone instance: %w", err) + } + if err := testcaseenv.VerifyStandaloneReady(ctx, deployment, name, standalone); err != nil { + return nil, fmt.Errorf("standalone not ready: %w", err) + } + return standalone, nil +} + +// DeployAndVerifyMonitoringConsole deploys a Monitoring Console and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyMonitoringConsole(ctx context.Context, deployment *Deployment, name string, licenseManagerRef string) (*enterpriseApi.MonitoringConsole, error) { + mc, err := deployment.DeployMonitoringConsole(ctx, name, licenseManagerRef) + if err != nil { + return nil, fmt.Errorf("unable to deploy Monitoring Console instance: %w", err) + } + if err := testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, name, mc); err != nil { + return nil, fmt.Errorf("monitoring console not ready: %w", err) + } + return mc, nil +} + +// VerifyIndexerCPULimits verifies CPU limits on all indexer pods in a single-site cluster +func (testcaseenv *TestCaseEnv) VerifyIndexerCPULimits(deployment *Deployment, indexerCount int, expectedCPULimit string) error { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deployment.GetName(), i) + if err := testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit); err != nil { + return err + } + } + return nil +} + +// VerifySearchHeadCPULimits verifies CPU limits on all search head pods +func (testcaseenv *TestCaseEnv) VerifySearchHeadCPULimits(deployment *Deployment, searchHeadCount int, expectedCPULimit string) error { + for i := 0; i < searchHeadCount; i++ { + podName := fmt.Sprintf(SearchHeadPod, deployment.GetName(), i) + if err := testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit); err != nil { + return err + } + } + return nil +} + +// VerifyM4ComponentsReady verifies the Cluster Manager, multisite indexers, multisite status, and SHC are ready. +func (testcaseenv *TestCaseEnv) VerifyM4ComponentsReady(ctx context.Context, deployment *Deployment, siteCount int, cmReadyFn func() error) error { + if err := cmReadyFn(); err != nil { + return err + } + if err := testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + if err := testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount); err != nil { + return err + } + return testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func (testcaseenv *TestCaseEnv) VerifyMCVersionChangedAndReady(ctx context.Context, deployment *Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) error { + if err := testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion); err != nil { + return err + } + return testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} + +// VerifyClusterReadyAndRFSF is a V4-only verification pattern that checks C3 cluster is ready (using ClusterManager) and RF/SF is met +func (testcaseenv *TestCaseEnv) VerifyClusterReadyAndRFSF(ctx context.Context, deployment *Deployment) error { + if err := testcaseenv.VerifyC3ClusterReady(ctx, deployment, testcaseenv.VerifyClusterManagerReady); err != nil { + return err + } + return testcaseenv.VerifyRFSFMet(ctx, deployment) +} + +// TriggerAndVerifyTelemetry is a common pattern for telemetry verification +func (testcaseenv *TestCaseEnv) TriggerAndVerifyTelemetry(ctx context.Context, deployment *Deployment, prevSubmissionTime string) error { + testcaseenv.TriggerTelemetrySubmission(ctx, deployment) + return testcaseenv.VerifyTelemetry(ctx, deployment, prevSubmissionTime) +} + +// VerifyProbeConfigAndScripts verifies probe config map exists and probe scripts are present on all pods. +// If includeStartup is true, the startup probe script is also checked. +func (testcaseenv *TestCaseEnv) VerifyProbeConfigAndScripts(ctx context.Context, deployment *Deployment, includeStartup bool) error { + testcaseenv.Log.Info("Get config map for livenessProbe and readinessProbe") + configMapName := enterprise.GetProbeConfigMapName(testcaseenv.GetName()) + _, err := GetConfigMap(ctx, deployment, testcaseenv.GetName(), configMapName) + if err != nil { + return fmt.Errorf("unable to get config map for livenessProbe and readinessProbe %s: %w", configMapName, err) + } + scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} + if includeStartup { + scriptsNames = append(scriptsNames, enterprise.GetStartupScriptName()) + } + allPods := DumpGetPods(testcaseenv.GetName()) + return testcaseenv.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) +} + +// NewStandaloneSpecWithMCRef creates a StandaloneSpec with a MonitoringConsoleRef set to the given MC name. +func NewStandaloneSpecWithMCRef(image string, mcName string) enterpriseApi.StandaloneSpec { + return NewStandaloneSpecWithMCRefAndResources(image, mcName, corev1.ResourceRequirements{}) +} + +// NewStandaloneSpecWithMCRefAndResources creates a StandaloneSpec with a MonitoringConsoleRef +// and custom resource requirements. +func NewStandaloneSpecWithMCRefAndResources(image string, mcName string, resources corev1.ResourceRequirements) enterpriseApi.StandaloneSpec { + return enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: image, + Resources: resources, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } +} + +// VerifyLMConfiguredOnMC verifies that the License Manager is configured on the Monitoring Console pod. +func VerifyLMConfiguredOnMC(ctx context.Context, deployment *Deployment) error { + monitoringConsolePodName := fmt.Sprintf(MonitoringConsolePod, deployment.GetName()) + return VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) +} + +// StandardC3Verification performs the standard V4-only set of verifications for a C3 cluster. +// This includes cluster ready (ClusterManager), RF/SF met, and monitoring console ready. +func (testcaseenv *TestCaseEnv) StandardC3Verification(ctx context.Context, deployment *Deployment, mc *enterpriseApi.MonitoringConsole) error { + if err := testcaseenv.VerifyClusterReadyAndRFSF(ctx, deployment); err != nil { + return err + } + return testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} + +// DeployAndVerifyC3 deploys a C3 single-site cluster and verifies it reaches the ready state. +func (c *ClusterReadinessConfig) DeployAndVerifyC3(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv, replicas int, shc bool, mcRef string) error { + if err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), replicas, shc, mcRef); err != nil { + return fmt.Errorf("unable to deploy C3 cluster: %w", err) + } + return c.VerifyC3ClusterReady(ctx, deployment, testcaseEnv) +} + +// DeployAndVerifyM4 deploys an M4 multisite cluster and verifies the Cluster Manager +// and all M4 components reach the ready state. +func (c *ClusterReadinessConfig) DeployAndVerifyM4(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv, indexerReplicas int, siteCount int, mcRef string) error { + if err := c.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), indexerReplicas, siteCount, mcRef); err != nil { + return fmt.Errorf("unable to deploy M4 cluster: %w", err) + } + return testcaseEnv.VerifyM4ComponentsReady(ctx, deployment, siteCount, func() error { + return c.ClusterManagerReady(ctx, deployment, testcaseEnv) + }) +} + +// DeployC3WithLicense sets up the license config map, deploys a C3 cluster, +// and verifies both the License Manager and cluster reach the ready state. +func (c *ClusterReadinessConfig) DeployC3WithLicense(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv, replicas int, shc bool, mcRef string) error { + if err := SetupLicenseConfigMap(ctx, testcaseEnv); err != nil { + return err + } + if err := c.DeployAndVerifyC3(ctx, deployment, testcaseEnv, replicas, shc, mcRef); err != nil { + return err + } + return c.LicenseManagerReady(ctx, deployment, testcaseEnv) +} + +// DeployM4WithLicense sets up the license config map, deploys an M4 multisite cluster, +// and verifies the License Manager, Cluster Manager, and all M4 components reach the ready state. +func (c *ClusterReadinessConfig) DeployM4WithLicense(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv, indexerReplicas int, siteCount int, mcRef string) error { + if err := SetupLicenseConfigMap(ctx, testcaseEnv); err != nil { + return err + } + if err := c.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), indexerReplicas, siteCount, mcRef); err != nil { + return fmt.Errorf("unable to deploy M4 cluster: %w", err) + } + if err := c.LicenseManagerReady(ctx, deployment, testcaseEnv); err != nil { + return err + } + return testcaseEnv.VerifyM4ComponentsReady(ctx, deployment, siteCount, func() error { + return c.ClusterManagerReady(ctx, deployment, testcaseEnv) + }) +} diff --git a/test/testenv/deployment.go b/test/testenv/deployment.go index f46f55171..60c558032 100644 --- a/test/testenv/deployment.go +++ b/test/testenv/deployment.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,7 +42,7 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" ) -// Deployment simply represents the deployment (standalone, clustered,...etc) we create on the testenv +// Deployment simply represents the deployment (Standalone, clustered, etc.) we create on the testenv type Deployment struct { name string testenv *TestCaseEnv @@ -110,7 +110,7 @@ func (d *Deployment) Teardown() error { var err error var output []byte - podName := GetOperatorPodName(d.testenv) + podName := d.testenv.GetOperatorPodName() if d.testenv.clusterWideOperator != "true" { output, err = exec.Command("kubectl", "logs", "-n", d.testenv.GetName(), podName).Output() } else { @@ -148,13 +148,13 @@ func (d *Deployment) Teardown() error { return cleanupErr } -// DeployStandalone deploys a standalone splunk enterprise instance on the specified testenv +// DeployStandalone deploys a Standalone Splunk Enterprise instance on the specified testenv func (d *Deployment) DeployStandalone(ctx context.Context, name string, mcRef string, LicenseManager string) (*enterpriseApi.Standalone, error) { standalone := newStandalone(name, d.testenv.namespace, d.testenv.splunkImage) // If license file specified, deploy License Manager if LicenseManager != "" && d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return nil, err @@ -196,7 +196,7 @@ func (d *Deployment) DeployMonitoringConsoleWithGivenSpec(ctx context.Context, n return deployed.(*enterpriseApi.MonitoringConsole), err } -// GetInstance retrieves the standalone, indexer, searchhead, LicenseManager instance +// GetInstance retrieves the Standalone, Indexer, Search Head, or License Manager instance func (d *Deployment) GetInstance(ctx context.Context, name string, instance client.Object) error { key := client.ObjectKey{Name: name, Namespace: d.testenv.namespace} @@ -343,7 +343,7 @@ func (d *Deployment) OperatorPodExecCommand(ctx context.Context, podName string, return stdout.String(), stderr.String(), nil } -// DeployLicenseManager deploys the license manager instance +// DeployLicenseManager deploys the License Manager instance func (d *Deployment) DeployLicenseManager(ctx context.Context, name string) (*enterpriseApi.LicenseManager, error) { if d.testenv.licenseFilePath == "" { @@ -361,7 +361,7 @@ func (d *Deployment) DeployLicenseManager(ctx context.Context, name string) (*en return deployed.(*enterpriseApi.LicenseManager), err } -// DeployLicenseMaster deploys the license manager instance +// DeployLicenseMaster deploys the License Master instance func (d *Deployment) DeployLicenseMaster(ctx context.Context, name string) (*enterpriseApiV3.LicenseMaster, error) { if d.testenv.licenseFilePath == "" { @@ -379,9 +379,9 @@ func (d *Deployment) DeployLicenseMaster(ctx context.Context, name string) (*ent return deployed.(*enterpriseApiV3.LicenseMaster), err } -// DeployClusterManager deploys the cluster manager +// DeployClusterManager deploys the Cluster Manager func (d *Deployment) DeployClusterManager(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name, "LicenseRef", LicenseManagerName) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name, "LicenseRef", LicenseManagerName) cm := newClusterManager(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { cm.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -399,9 +399,9 @@ func (d *Deployment) DeployClusterManager(ctx context.Context, name, LicenseMana return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMaster deploys the cluster manager +// DeployClusterMaster deploys the Cluster Master func (d *Deployment) DeployClusterMaster(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-master", "name", name, "LicenseRef", LicenseManagerName) + d.testenv.Log.Info("Deploying Cluster Master", "name", name, "LicenseRef", LicenseManagerName) cm := newClusterMaster(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { cm.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -419,9 +419,9 @@ func (d *Deployment) DeployClusterMaster(ctx context.Context, name, LicenseManag return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeployClusterManagerWithSmartStoreIndexes deploys the cluster manager with smartstore indexes +// DeployClusterManagerWithSmartStoreIndexes deploys the Cluster Manager with SmartStore indexes func (d *Deployment) DeployClusterManagerWithSmartStoreIndexes(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, smartstorespec enterpriseApi.SmartStoreSpec) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterManagerWithGivenIndexes(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage, smartstorespec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -433,9 +433,9 @@ func (d *Deployment) DeployClusterManagerWithSmartStoreIndexes(ctx context.Conte return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMasterWithSmartStoreIndexes deploys the cluster manager with smartstore indexes +// DeployClusterMasterWithSmartStoreIndexes deploys the Cluster Master with SmartStore indexes func (d *Deployment) DeployClusterMasterWithSmartStoreIndexes(ctx context.Context, name, LicenseManagerName, ansibleConfig string, smartstorespec enterpriseApi.SmartStoreSpec) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterMasterWithGivenIndexes(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage, smartstorespec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -447,12 +447,12 @@ func (d *Deployment) DeployClusterMasterWithSmartStoreIndexes(ctx context.Contex return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeployIndexerCluster deploys the indexer cluster +// DeployIndexerCluster deploys the Indexer Cluster func (d *Deployment) DeployIndexerCluster(ctx context.Context, name, LicenseManagerName string, count int, clusterManagerRef string, ansibleConfig string, queue, os corev1.ObjectReference, serviceAccountName string) (*enterpriseApi.IndexerCluster, error) { - d.testenv.Log.Info("Deploying indexer cluster", "name", name, "CM", clusterManagerRef) + d.testenv.Log.Info("Deploying Indexer Cluster", "name", name, "CM", clusterManagerRef) indexer := newIndexerCluster(name, d.testenv.namespace, LicenseManagerName, count, clusterManagerRef, ansibleConfig, d.testenv.splunkImage, queue, os, serviceAccountName) pdata, _ := json.Marshal(indexer) - d.testenv.Log.Info("indexer cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Indexer Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, name, indexer) if err != nil { return nil, err @@ -461,14 +461,14 @@ func (d *Deployment) DeployIndexerCluster(ctx context.Context, name, LicenseMana return deployed.(*enterpriseApi.IndexerCluster), err } -// DeployIngestorCluster deploys the ingestor cluster +// DeployIngestorCluster deploys the Ingestor Cluster func (d *Deployment) DeployIngestorCluster(ctx context.Context, name string, count int, queue, os corev1.ObjectReference, serviceAccountName string) (*enterpriseApi.IngestorCluster, error) { - d.testenv.Log.Info("Deploying ingestor cluster", "name", name) + d.testenv.Log.Info("Deploying Ingestor Cluster", "name", name) ingestor := newIngestorCluster(name, d.testenv.namespace, count, d.testenv.splunkImage, queue, os, serviceAccountName) pdata, _ := json.Marshal(ingestor) - d.testenv.Log.Info("ingestor cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Ingestor Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, name, ingestor) if err != nil { return nil, err @@ -509,13 +509,13 @@ func (d *Deployment) DeployObjectStorage(ctx context.Context, name string, objSt return deployed.(*enterpriseApi.ObjectStorage), err } -// DeployIngestorClusterWithAdditionalConfiguration deploys the ingestor cluster with additional configuration +// DeployIngestorClusterWithAdditionalConfiguration deploys the Ingestor Cluster with additional configuration func (d *Deployment) DeployIngestorClusterWithAdditionalConfiguration(ctx context.Context, ic *enterpriseApi.IngestorCluster) (*enterpriseApi.IngestorCluster, error) { - d.testenv.Log.Info("Deploying ingestor cluster with additional configuration", "name", ic.Name) + d.testenv.Log.Info("Deploying Ingestor Cluster with additional configuration", "name", ic.Name) pdata, _ := json.Marshal(ic) - d.testenv.Log.Info("ingestor cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Ingestor Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, ic.Name, ic) if err != nil { return nil, err @@ -524,9 +524,9 @@ func (d *Deployment) DeployIngestorClusterWithAdditionalConfiguration(ctx contex return deployed.(*enterpriseApi.IngestorCluster), err } -// DeploySearchHeadCluster deploys a search head cluster +// DeploySearchHeadCluster deploys a Search Head Cluster func (d *Deployment) DeploySearchHeadCluster(ctx context.Context, name, ClusterManagerRef, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApi.SearchHeadCluster, error) { - d.testenv.Log.Info("Deploying search head cluster", "name", name) + d.testenv.Log.Info("Deploying Search Head Cluster", "name", name) sh := newSearchHeadCluster(name, d.testenv.namespace, ClusterManagerRef, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { sh.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -744,7 +744,7 @@ func (d *Deployment) DeleteCR(ctx context.Context, cr client.Object) error { return err } -// DeploySingleSiteCluster deploys a lm and indexer cluster (shc optional) +// DeploySingleSiteCluster deploys a LM and Indexer Cluster (SHC optional) func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, indexerReplicas int, shc bool, mcRef string) error { var LicenseManager string @@ -753,13 +753,13 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i if d.testenv.licenseFilePath != "" { // Enable LM to be tested if strings.Contains(name, "master") { - // Deploy the license master + // Deploy the License Master _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err } } else { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -770,26 +770,26 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i } if strings.Contains(name, "master") { - // Deploy the cluster master + // Deploy the Cluster Master _, err := d.DeployClusterMaster(ctx, name, LicenseManager, "", mcRef) if err != nil { return err } } else { - // Deploy the cluster manager + // Deploy the Cluster Manager _, err := d.DeployClusterManager(ctx, name, LicenseManager, "", mcRef) if err != nil { return err } } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err := d.DeployIndexerCluster(ctx, name+"-idxc", LicenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err } - // Deploy the SH cluster + // Deploy the SH Cluster if shc { _, err = d.DeploySearchHeadCluster(ctx, name+"-shc", name, LicenseManager, "", mcRef) if err != nil { @@ -800,7 +800,7 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i return nil } -// DeployMultisiteClusterMasterWithSearchHead deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterMasterWithSearchHead deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseMaster string @@ -815,7 +815,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Cont LicenseMaster = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -862,7 +862,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Cont return nil } -// DeployMultisiteClusterWithSearchHead deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterWithSearchHead deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseManager string @@ -871,13 +871,13 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n if d.testenv.licenseFilePath != "" { // Enable LM to be tested if strings.Contains(name, "master") { - // Deploy the license master + // Deploy the License Master _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err } } else { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -887,7 +887,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -934,14 +934,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n return nil } -// DeployMultisiteCluster deploys a lm, cluster-manager, indexers in multiple sites +// DeployMultisiteCluster deploys a LM, Cluster Manager, and indexers in multiple sites func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -950,7 +950,7 @@ func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, in LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -984,13 +984,13 @@ func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, in return nil } -// DeployStandaloneWithLMaster deploys a standalone splunk enterprise instance with license manager on the specified testenv +// DeployStandaloneWithLMaster deploys a Standalone Splunk Enterprise instance with License Master on the specified testenv func (d *Deployment) DeployStandaloneWithLMaster(ctx context.Context, name string, mcRef string) (*enterpriseApi.Standalone, error) { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return nil, err @@ -1011,13 +1011,13 @@ func (d *Deployment) DeployStandaloneWithLMaster(ctx context.Context, name strin return deployed.(*enterpriseApi.Standalone), err } -// DeployStandaloneWithLM deploys a standalone splunk enterprise instance with license manager on the specified testenv +// DeployStandaloneWithLM deploys a Standalone Splunk Enterprise instance with License Manager on the specified testenv func (d *Deployment) DeployStandaloneWithLM(ctx context.Context, name string, mcRef string) (*enterpriseApi.Standalone, error) { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return nil, err @@ -1038,7 +1038,7 @@ func (d *Deployment) DeployStandaloneWithLM(ctx context.Context, name string, mc return deployed.(*enterpriseApi.Standalone), err } -// DeployStandaloneWithGivenSpec deploys a standalone with given spec +// DeployStandaloneWithGivenSpec deploys a Standalone with the given spec func (d *Deployment) DeployStandaloneWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.StandaloneSpec) (*enterpriseApi.Standalone, error) { standalone := newStandaloneWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, standalone) @@ -1048,7 +1048,7 @@ func (d *Deployment) DeployStandaloneWithGivenSpec(ctx context.Context, name str return deployed.(*enterpriseApi.Standalone), err } -// DeployStandaloneWithGivenSmartStoreSpec deploys a standalone give smartstore spec +// DeployStandaloneWithGivenSmartStoreSpec deploys a Standalone with the given SmartStore spec func (d *Deployment) DeployStandaloneWithGivenSmartStoreSpec(ctx context.Context, name string, smartStoreSpec enterpriseApi.SmartStoreSpec) (*enterpriseApi.Standalone, error) { spec := enterpriseApi.StandaloneSpec{ @@ -1070,14 +1070,14 @@ func (d *Deployment) DeployStandaloneWithGivenSmartStoreSpec(ctx context.Context return deployed.(*enterpriseApi.Standalone), err } -// DeployMultisiteClusterWithSearchHeadAndIndexes deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterWithSearchHeadAndIndexes deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context.Context, name string, indexerReplicas int, siteCount int, indexesSecret string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1086,7 +1086,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context. LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1125,14 +1125,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context. return err } -// DeployMultisiteClusterMasterWithSearchHeadAndIndexes deploys a lm, cluster-master, indexers in multiple sites and SH clusters +// DeployMultisiteClusterMasterWithSearchHeadAndIndexes deploys a LM, Cluster Master, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx context.Context, name string, indexerReplicas int, siteCount int, indexesSecret string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1141,7 +1141,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx co LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1180,9 +1180,9 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx co return err } -// DeployClusterManagerWithGivenSpec deploys the cluster manager with given SPEC +// DeployClusterManagerWithGivenSpec deploys the Cluster Manager with given SPEC func (d *Deployment) DeployClusterManagerWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.ClusterManagerSpec) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterManagerWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -1191,9 +1191,9 @@ func (d *Deployment) DeployClusterManagerWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMasterWithGivenSpec deploys the cluster manager with given SPEC +// DeployClusterMasterWithGivenSpec deploys the Cluster Master with given SPEC func (d *Deployment) DeployClusterMasterWithGivenSpec(ctx context.Context, name string, spec enterpriseApiV3.ClusterMasterSpec) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-master", "name", name) + d.testenv.Log.Info("Deploying Cluster Master", "name", name) cm := newClusterMasterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -1202,17 +1202,17 @@ func (d *Deployment) DeployClusterMasterWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeploySearchHeadClusterWithGivenSpec deploys a search head cluster +// DeploySearchHeadClusterWithGivenSpec deploys a Search Head Cluster func (d *Deployment) DeploySearchHeadClusterWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.SearchHeadClusterSpec) (*enterpriseApi.SearchHeadCluster, error) { - d.testenv.Log.Info("Deploying search head cluster", "name", name) + d.testenv.Log.Info("Deploying Search Head Cluster", "name", name) indexer := newSearchHeadClusterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, indexer) return deployed.(*enterpriseApi.SearchHeadCluster), err } -// DeployLicenseManagerWithGivenSpec deploys the license manager with given SPEC +// DeployLicenseManagerWithGivenSpec deploys the License Manager with given SPEC func (d *Deployment) DeployLicenseManagerWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.LicenseManagerSpec) (*enterpriseApi.LicenseManager, error) { - d.testenv.Log.Info("Deploying license-manager", "name", name) + d.testenv.Log.Info("Deploying License Manager", "name", name) lm := newLicenseManagerWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, lm) if err != nil { @@ -1221,9 +1221,9 @@ func (d *Deployment) DeployLicenseManagerWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApi.LicenseManager), err } -// DeployLicenseMasterWithGivenSpec deploys the license manager with given SPEC +// DeployLicenseMasterWithGivenSpec deploys the License Master with given SPEC func (d *Deployment) DeployLicenseMasterWithGivenSpec(ctx context.Context, name string, spec enterpriseApiV3.LicenseMasterSpec) (*enterpriseApiV3.LicenseMaster, error) { - d.testenv.Log.Info("Deploying license-master", "name", name) + d.testenv.Log.Info("Deploying License Master", "name", name) lm := newLicenseMasterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, lm) if err != nil { @@ -1232,7 +1232,7 @@ func (d *Deployment) DeployLicenseMasterWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApiV3.LicenseMaster), err } -// DeploySingleSiteClusterWithGivenAppFrameworkSpec deploys indexer cluster (lm, shc optional) with app framework spec +// DeploySingleSiteClusterWithGivenAppFrameworkSpec deploys Indexer Cluster (LM, SHC optional) with app framework spec func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx context.Context, name string, indexerReplicas int, shc bool, appFrameworkSpecIdxc enterpriseApi.AppFrameworkSpec, appFrameworkSpecShc enterpriseApi.AppFrameworkSpec, mcName string, licenseManager string) (*enterpriseApi.ClusterManager, *enterpriseApi.IndexerCluster, *enterpriseApi.SearchHeadCluster, error) { cm := &enterpriseApi.ClusterManager{} @@ -1241,14 +1241,14 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster manager + // Deploy the Cluster Manager cmSpec := enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1270,7 +1270,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex return cm, idxc, sh, err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster idxc, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return cm, idxc, sh, err @@ -1298,7 +1298,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex } pdata, _ := json.Marshal(shSpec) - d.testenv.Log.Info("Search head Spec", "cr", pdata) + d.testenv.Log.Info("Search Head Spec", "cr", pdata) if shc { sh, err = d.DeploySearchHeadClusterWithGivenSpec(ctx, name+"-shc", shSpec) @@ -1310,7 +1310,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex return cm, idxc, sh, nil } -// DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec deploys indexer cluster (lm, shc optional) with app framework spec +// DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec deploys Indexer Cluster (LM, SHC optional) with app framework spec func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx context.Context, name string, indexerReplicas int, shc bool, appFrameworkSpecIdxc enterpriseApi.AppFrameworkSpec, appFrameworkSpecShc enterpriseApi.AppFrameworkSpec, mcName string, licenseMaster string) (*enterpriseApiV3.ClusterMaster, *enterpriseApi.IndexerCluster, *enterpriseApi.SearchHeadCluster, error) { cm := &enterpriseApiV3.ClusterMaster{} @@ -1319,14 +1319,14 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster manager + // Deploy the Cluster Master cmSpec := enterpriseApiV3.ClusterMasterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1348,7 +1348,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx return cm, idxc, sh, err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster idxc, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseMaster, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return cm, idxc, sh, err @@ -1376,7 +1376,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx } pdata, _ := json.Marshal(shSpec) - d.testenv.Log.Info("Search head Spec", "cr", pdata) + d.testenv.Log.Info("Search Head Spec", "cr", pdata) if shc { sh, err = d.DeploySearchHeadClusterWithGivenSpec(ctx, name+"-shc", shSpec) @@ -1397,14 +1397,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx con // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, licenseManager) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1459,7 +1459,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx con multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-manager") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1501,14 +1501,14 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, licenseMaster) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1563,7 +1563,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-master") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1596,14 +1596,14 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c return cm, idxc, sh, nil } -// DeploySingleSiteClusterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console +// DeploySingleSiteClusterWithGivenMonitoringConsole deploys Indexer Cluster (LM, SHC optional) with given Monitoring Console func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx context.Context, name string, indexerReplicas int, shc bool, monitoringConsoleName string) error { licenseManager := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1612,7 +1612,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte licenseManager = name } - // Deploy the cluster manager + // Deploy the Cluster Manager cmSpec := enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1633,7 +1633,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte return err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err @@ -1668,14 +1668,14 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte return nil } -// DeploySingleSiteClusterMasterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console +// DeploySingleSiteClusterMasterWithGivenMonitoringConsole deploys Indexer Cluster (LM, SHC optional) with given Monitoring Console func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx context.Context, name string, indexerReplicas int, shc bool, monitoringConsoleName string) error { licenseMaster := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1684,7 +1684,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx licenseMaster = name } - // Deploy the cluster manager + // Deploy the Cluster Master cmSpec := enterpriseApiV3.ClusterMasterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1705,7 +1705,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx return err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseMaster, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err @@ -1740,14 +1740,14 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx return nil } -// DeployMultisiteClusterWithMonitoringConsole deploys cluster-manager, indexers in multiple sites (SHC LM Optional) with monitoring console +// DeployMultisiteClusterWithMonitoringConsole deploys Cluster Manager, Indexers in multiple sites (SHC LM Optional) with Monitoring Console func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Context, name string, indexerReplicas int, siteCount int, monitoringConsoleName string, shc bool) error { licenseManager := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1756,7 +1756,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con licenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1810,7 +1810,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-manager") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1840,14 +1840,14 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con return nil } -// DeployMultisiteClusterMasterWithMonitoringConsole deploys cluster-master, indexers in multiple sites (SHC LM Optional) with monitoring console +// DeployMultisiteClusterMasterWithMonitoringConsole deploys Cluster Master, Indexers in multiple sites (SHC LM Optional) with Monitoring Console func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx context.Context, name string, indexerReplicas int, siteCount int, monitoringConsoleName string, shc bool) error { licenseMaster := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1856,7 +1856,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte licenseMaster = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1910,7 +1910,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-master") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1949,3 +1949,14 @@ func (d *Deployment) GetConfigMap(ctx context.Context, name string) (*corev1.Con } return cm, nil } + +// GetAndDeleteCR fetches a CR by name and deletes it, returning any error encountered. +func GetAndDeleteCR(ctx context.Context, deployment *Deployment, obj client.Object, instanceName string) error { + if err := deployment.GetInstance(ctx, instanceName, obj); err != nil { + return fmt.Errorf("unable to get instance %s: %w", instanceName, err) + } + if err := deployment.DeleteCR(ctx, obj); err != nil { + return fmt.Errorf("unable to delete instance %s: %w", instanceName, err) + } + return nil +} diff --git a/test/testenv/filespec.go b/test/testenv/filespec.go index 8e9131955..3dddb158b 100644 --- a/test/testenv/filespec.go +++ b/test/testenv/filespec.go @@ -99,11 +99,6 @@ func (p remotePath) Join(elem pathSpec) remotePath { return newRemotePath(path.Join(p.file, elem.String())) } -func (p remotePath) StripShortcuts() remotePath { - p = p.Clean() - return newRemotePath(stripPathShortcuts(p.file)) -} - func (p remotePath) StripSlashes() remotePath { return newRemotePath(stripLeadingSlash(p.file)) } @@ -123,25 +118,3 @@ func stripLeadingSlash(file string) string { // tar strips the leading '/' and '\' if it's there, so we will too return strings.TrimLeft(file, `/\`) } - -// stripPathShortcuts removes any leading or trailing "../" from a given path -func stripPathShortcuts(p string) string { - newPath := p - trimmed := strings.TrimPrefix(newPath, "../") - - for trimmed != newPath { - newPath = trimmed - trimmed = strings.TrimPrefix(newPath, "../") - } - - // trim leftover {".", ".."} - if newPath == "." || newPath == ".." { - newPath = "" - } - - if len(newPath) > 0 && string(newPath[0]) == "/" { - return newPath[1:] - } - - return newPath -} diff --git a/test/testenv/gcputils.go b/test/testenv/gcputils.go index 78c41d1aa..e6b9a0b1e 100644 --- a/test/testenv/gcputils.go +++ b/test/testenv/gcputils.go @@ -120,9 +120,9 @@ func CheckPrefixExistsOnGCP(prefix string) bool { logf.Log.Error(err, "Error listing objects in GCP bucket") return false } - logf.Log.Info("CHECKING OBJECT", "OBJECT", objAttrs.Name) + logf.Log.Info("CHECKING OBJECT", "object", objAttrs.Name) if strings.Contains(objAttrs.Name, prefix) { - logf.Log.Info("Prefix found in bucket", "Prefix", prefix, "Object", objAttrs.Name) + logf.Log.Info("Prefix found in bucket", "prefix", prefix, "object", objAttrs.Name) return true } } @@ -146,12 +146,12 @@ func CreateBucketAndPathIfNotExist(bucketName, path string) error { // Create the bucket err = client.Client.Bucket(bucketName).Create(ctx, gcpProjectID, nil) if err != nil { - logf.Log.Error(err, "Failed to create bucket", "Bucket Name", bucketName) + logf.Log.Error(err, "Failed to create bucket", "bucketName", bucketName) return err } - logf.Log.Info("Bucket created", "Bucket Name", bucketName) + logf.Log.Info("Bucket created", "bucketName", bucketName) } else if err != nil { - logf.Log.Error(err, "Error checking bucket attributes", "Bucket Name", bucketName) + logf.Log.Error(err, "Error checking bucket attributes", "bucketName", bucketName) return err } @@ -161,16 +161,16 @@ func CreateBucketAndPathIfNotExist(bucketName, path string) error { // Create a zero-length object to represent the path wc := client.Client.Bucket(bucketName).Object(path).NewWriter(ctx) if _, err := wc.Write([]byte{}); err != nil { - logf.Log.Error(err, "Failed to create path", "Path", path) + logf.Log.Error(err, "Failed to create path", "path", path) return err } if err := wc.Close(); err != nil { - logf.Log.Error(err, "Failed to finalize path creation", "Path", path) + logf.Log.Error(err, "Failed to finalize path creation", "path", path) return err } - logf.Log.Info("Path created", "Path", path) + logf.Log.Info("Path created", "path", path) } else if err != nil { - logf.Log.Error(err, "Error checking path attributes", "Path", path) + logf.Log.Error(err, "Error checking path attributes", "path", path) return err } @@ -209,7 +209,7 @@ func DownloadFileFromGCP(bucketName, objectName, gcpFilePath, downloadDir string objectPath := filepath.Join(gcpFilePath, objectName) rc, err := client.Client.Bucket(bucketName).Object(objectPath).NewReader(ctx) if err != nil { - logf.Log.Error(err, "Failed to create reader for object", "Object", objectName) + logf.Log.Error(err, "Failed to create reader for object", "object", objectName) return "", err } defer rc.Close() @@ -217,14 +217,14 @@ func DownloadFileFromGCP(bucketName, objectName, gcpFilePath, downloadDir string localPath := filepath.Join(downloadDir, objectName) file, err := os.Create(localPath) if err != nil { - logf.Log.Error(err, "Failed to create local file", "Filename", localPath) + logf.Log.Error(err, "Failed to create local file", "filename", localPath) return "", err } defer file.Close() written, err := io.Copy(file, rc) if err != nil { - logf.Log.Error(err, "Failed to download object", "Object", objectName) + logf.Log.Error(err, "Failed to download object", "object", objectName) return "", err } @@ -319,18 +319,18 @@ func DeleteFileOnGCP(bucketName, objectName string) error { err = client.Client.Bucket(bucketName).Object(objectName).Delete(ctx) if err != nil && err != storage.ErrObjectNotExist { - logf.Log.Error(err, "Unable to delete object from bucket", "Object Name", objectName, "Bucket Name", bucketName) + logf.Log.Error(err, "Unable to delete object from bucket", "objectName", objectName, "bucketName", bucketName) return err } // Optionally, verify deletion _, err = client.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) if err == storage.ErrObjectNotExist { - logf.Log.Info("Deleted file on GCP", "File Name", objectName, "Bucket", bucketName) + logf.Log.Info("Deleted file on GCP", "fileName", objectName, "bucket", bucketName) return nil } if err != nil { - logf.Log.Error(err, "Error verifying deletion of object", "Object Name", objectName, "Bucket Name", bucketName) + logf.Log.Error(err, "Error verifying deletion of object", "objectName", objectName, "bucketName", bucketName) return err } @@ -342,12 +342,12 @@ func GetFilesInPathOnGCP(bucketName, path string) []string { resp := GetFileListOnGCP(bucketName, path) var files []string for _, obj := range resp { - logf.Log.Info("CHECKING OBJECT", "OBJECT", obj.Name) + logf.Log.Info("CHECKING OBJECT", "object", obj.Name) if strings.HasPrefix(obj.Name, path) { filename := strings.TrimPrefix(obj.Name, path) // This condition filters out directories as GCP returns objects with their full paths if len(filename) > 1 && !strings.HasSuffix(filename, "/") { - logf.Log.Info("File found in bucket", "Path", path, "Object", obj.Name) + logf.Log.Info("File found in bucket", "path", path, "object", obj.Name) files = append(files, filename) } } @@ -358,10 +358,10 @@ func GetFilesInPathOnGCP(bucketName, path string) []string { // DownloadFilesFromGCP downloads a list of files from a GCP bucket to a local directory func DownloadFilesFromGCP(bucketName, gcpAppDir, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading file from GCP", "File name", key) + logf.Log.Info("Downloading file from GCP", "fileName", key) _, err := DownloadFileFromGCP(bucketName, key, gcpAppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -372,22 +372,22 @@ func DownloadFilesFromGCP(bucketName, gcpAppDir, downloadDir string, appList []s func UploadFilesToGCP(bucketName, gcpTestDir string, appList []string, uploadDir string) ([]string, error) { var uploadedFiles []string for _, key := range appList { - logf.Log.Info("Uploading file to GCP", "File name", key) - logf.Log.Info("Using bucket", "Bucket", bucketName, "Path", gcpTestDir, "Upload Dir", uploadDir) + logf.Log.Info("Uploading file to GCP", "fileName", key) + logf.Log.Info("Using bucket", "bucket", bucketName, "path", gcpTestDir, "uploadDir", uploadDir) fileLocation := filepath.Join(uploadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open file", "File name", key) + logf.Log.Error(err, "Unable to open file", "fileName", key) return nil, err } defer fileBody.Close() objectPath, err := UploadFileToGCP(bucketName, key, gcpTestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } - logf.Log.Info("File uploaded to GCP", "File name", objectPath) + logf.Log.Info("File uploaded to GCP", "fileName", objectPath) uploadedFiles = append(uploadedFiles, objectPath) } return uploadedFiles, nil diff --git a/test/testenv/index_ingest_separation_utils.go b/test/testenv/index_ingest_separation_utils.go new file mode 100644 index 000000000..d87ac9366 --- /dev/null +++ b/test/testenv/index_ingest_separation_utils.go @@ -0,0 +1,108 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package testenv + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" +) + +// DeployQueueAndObjectStorage deploys a Queue and ObjectStorage CR and returns both. +func DeployQueueAndObjectStorage(ctx context.Context, deployment *Deployment, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec) (*enterpriseApi.Queue, *enterpriseApi.ObjectStorage, error) { + q, err := deployment.DeployQueue(ctx, "queue", qSpec) + if err != nil { + return nil, nil, fmt.Errorf("unable to deploy Queue: %w", err) + } + + objStorage, err := deployment.DeployObjectStorage(ctx, "os", osSpec) + if err != nil { + return nil, nil, fmt.Errorf("unable to deploy ObjectStorage: %w", err) + } + + return q, objStorage, nil +} + +// SetupIngestorStack deploys the full Queue/ObjectStorage/IngestorCluster/ClusterManager/IndexerCluster stack +// and verifies each component reaches the Ready phase. +func (testcaseEnvInst *TestCaseEnv) SetupIngestorStack(ctx context.Context, deployment *Deployment, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec, cmSpec enterpriseApi.ClusterManagerSpec) error { + volumeSpec := []enterpriseApi.SQSVolumeSpec{GenerateQueueVolumeSpec( + "queue-secret-ref-volume", + testcaseEnvInst.GetIndexIngestSepSecretName(), + )} + qSpec.SQS.VolList = volumeSpec + + q, objStorage, err := DeployQueueAndObjectStorage(ctx, deployment, qSpec, osSpec) + if err != nil { + return err + } + + if _, err := deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, ""); err != nil { + return fmt.Errorf("unable to deploy Ingestor Cluster: %w", err) + } + + if _, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec); err != nil { + return fmt.Errorf("unable to deploy Cluster Manager: %w", err) + } + + if _, err := deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, ""); err != nil { + return fmt.Errorf("unable to deploy Indexer Cluster: %w", err) + } + + if err := testcaseEnvInst.VerifyIngestorReady(ctx, deployment); err != nil { + return fmt.Errorf("ingestor not ready: %w", err) + } + if err := testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment); err != nil { + return fmt.Errorf("cluster manager not ready: %w", err) + } + if err := testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment); err != nil { + return fmt.Errorf("indexers not ready: %w", err) + } + return nil +} + +// VerifyCredentialSecretVersion checks that a credential secret version is set and valid (not empty or "0"). +func VerifyCredentialSecretVersion(version string, label string) error { + if version == "" { + return fmt.Errorf("%s queue status credential access secret version is empty", label) + } + if version == "0" { + return fmt.Errorf("%s queue status credential access secret version is 0", label) + } + return nil +} + +// DeleteIngestorStack tears down the full Queue/ObjectStorage/IngestorCluster/IndexerCluster stack. +func DeleteIngestorStack(ctx context.Context, deployment *Deployment) error { + // Delete the Indexer Cluster + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.IndexerCluster{}, deployment.GetName()+"-idxc"); err != nil { + return err + } + + // Delete the Ingestor Cluster + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.IngestorCluster{}, deployment.GetName()+"-ingest"); err != nil { + return err + } + + // Delete the Queue + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.Queue{}, "queue"); err != nil { + return err + } + + // Delete the ObjectStorage + return GetAndDeleteCR(ctx, deployment, &enterpriseApi.ObjectStorage{}, "os") +} diff --git a/test/testenv/ingest_utils.go b/test/testenv/ingest_utils.go index 2c0403b1e..2f1a7eb91 100644 --- a/test/testenv/ingest_utils.go +++ b/test/testenv/ingest_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -90,7 +90,7 @@ func CreateMockLogfile(logFile string, totalLines int) error { } // CreateAnIndexStandalone creates an index on a standalone instance using the CLI -func CreateAnIndexStandalone(ctx context.Context, indexName string, podName string, deployment *Deployment) error { +func CreateAnIndexStandalone(ctx context.Context, deployment *Deployment, indexName string, podName string) error { var addIndexCmd strings.Builder splunkBin := "/opt/splunk/bin/splunk" @@ -120,10 +120,10 @@ func CreateAnIndexStandalone(ctx context.Context, indexName string, podName stri } // IngestFileViaOneshot ingests a file into an instance using the oneshot CLI -func IngestFileViaOneshot(ctx context.Context, logFile string, indexName string, podName string, deployment *Deployment) error { +func IngestFileViaOneshot(ctx context.Context, deployment *Deployment, logFile string, indexName string, podName string) error { // Send it to the instance - resp, stderr, cpErr := CopyFileToPod(ctx, podName, logFile, logFile, deployment) + resp, stderr, cpErr := CopyFileToPod(ctx, deployment, podName, logFile, logFile) if cpErr != nil { logf.Log.Error(cpErr, "Failed File Copy to pod", "logFile", logFile, "podName", podName, "stderr", stderr) return cpErr @@ -158,7 +158,7 @@ func IngestFileViaOneshot(ctx context.Context, logFile string, indexName string, } // CopyFileToPod copies a file locally from srcPath to the destPath on the pod specified in podName -func CopyFileToPod(ctx context.Context, podName string, srcPath string, destPath string, deployment *Deployment) (string, string, error) { +func CopyFileToPod(ctx context.Context, deployment *Deployment, podName string, srcPath string, destPath string) (string, string, error) { // Create tar file stream reader, writer := io.Pipe() if destPath != "/" && strings.HasSuffix(string(destPath[len(destPath)-1]), "/") { @@ -224,10 +224,10 @@ func CopyFileToPod(ctx context.Context, podName string, srcPath string, destPath } // IngestFileViaMonitor ingests a file into an instance using the monitor CLI -func IngestFileViaMonitor(ctx context.Context, logFile string, indexName string, podName string, deployment *Deployment) error { +func IngestFileViaMonitor(ctx context.Context, deployment *Deployment, logFile string, indexName string, podName string) error { // Send it to the instance - resp, stderr, cpErr := CopyFileToPod(ctx, podName, logFile, logFile, deployment) + resp, stderr, cpErr := CopyFileToPod(ctx, deployment, podName, logFile, logFile) if cpErr != nil { logf.Log.Error(cpErr, "Failed File Copy to pod", "logFile", logFile, "podName", podName, "stderr", stderr) return cpErr diff --git a/test/testenv/lmutil.go b/test/testenv/lmutil.go index 9f7aa5dd7..8d8b8e3d4 100644 --- a/test/testenv/lmutil.go +++ b/test/testenv/lmutil.go @@ -1,27 +1,29 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package testenv import ( "context" "encoding/json" + "fmt" "strings" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + corev1 "k8s.io/api/core/v1" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -56,6 +58,143 @@ func CheckLicenseManagerConfigured(ctx context.Context, deployment *Deployment, return false } licenseManager := restResponse.Entry[0].Content.ManagerURI - logf.Log.Info("License Manager configuration on POD", "POD", podName, "License Manager", licenseManager) + logf.Log.Info("License Manager configuration on POD", "pod", podName, "licenseManager", licenseManager) return strings.Contains(licenseManager, "license-manager-service:8089") || strings.Contains(licenseManager, "license-master-service:8089") } + +// LicenseTestConfig holds the version-specific (V3/V4) deployment and verification +// callbacks used by the license manager test functions. +// +// The spec parameters use interface{} because V3 (LicenseMasterSpec) and V4 +// (LicenseManagerSpec) are distinct types with no shared interface. Callers +// must pass the correct concrete type for the API version in use; a mismatch +// will cause a runtime panic on the type assertion inside the callback. +type LicenseTestConfig struct { + *ClusterReadinessConfig + DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *Deployment, name string, spec interface{}) (interface{}, error) + BuildLMAppFrameworkSpec func(testcaseEnvInst *TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} + LicenseManagerPodName string + CrKind string +} + +// MasterManagerLMTestConfig pairs a name prefix and test label with a factory +// function that returns the appropriate LicenseTestConfig. +// It mirrors MasterManagerTestConfig (in common_test_patterns.go) but returns +// *LicenseTestConfig instead of *ClusterReadinessConfig because license-manager +// tests need the extra deployment callbacks that LicenseTestConfig provides. +type MasterManagerLMTestConfig struct { + NamePrefix string + Label string + NewConfig func() *LicenseTestConfig +} + +// NewLicenseCommonSplunkSpec returns a CommonSplunkSpec pre-configured with the +// license config map volume and license URL. +func NewLicenseCommonSplunkSpec(testcaseEnvInst *TestCaseEnv) enterpriseApi.CommonSplunkSpec { + return enterpriseApi.CommonSplunkSpec{ + Volumes: []corev1.Volume{{ + Name: "licenses", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: testcaseEnvInst.GetLMConfigMap(), + }, + }, + }, + }}, + LicenseURL: "/mnt/licenses/enterprise.lic", + Spec: enterpriseApi.Spec{ImagePullPolicy: "Always", Image: testcaseEnvInst.GetSplunkImage()}, + } +} + +// NewLicenseMasterConfig returns a LicenseTestConfig for V3 (LicenseMaster) tests. +func NewLicenseMasterConfig() *LicenseTestConfig { + return &LicenseTestConfig{ + ClusterReadinessConfig: NewClusterReadinessConfigV3(), + DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *Deployment, name string, spec interface{}) (interface{}, error) { + return deployment.DeployLicenseMasterWithGivenSpec(ctx, name, spec.(enterpriseApiV3.LicenseMasterSpec)) + }, + BuildLMAppFrameworkSpec: func(testcaseEnvInst *TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApiV3.LicenseMasterSpec{ + CommonSplunkSpec: NewLicenseCommonSplunkSpec(testcaseEnvInst), + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: LicenseMasterPod, + CrKind: "LicenseMaster", + } +} + +// NewLicenseManagerConfig returns a LicenseTestConfig for V4 (LicenseManager) tests. +func NewLicenseManagerConfig() *LicenseTestConfig { + return &LicenseTestConfig{ + ClusterReadinessConfig: NewClusterReadinessConfigV4(), + DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *Deployment, name string, spec interface{}) (interface{}, error) { + return deployment.DeployLicenseManagerWithGivenSpec(ctx, name, spec.(enterpriseApi.LicenseManagerSpec)) + }, + BuildLMAppFrameworkSpec: func(testcaseEnvInst *TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApi.LicenseManagerSpec{ + CommonSplunkSpec: NewLicenseCommonSplunkSpec(testcaseEnvInst), + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: LicenseManagerPod, + CrKind: "LicenseManager", + } +} + +// DownloadAppFiles downloads app files from the appropriate cloud provider. +func DownloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) error { + var err error + + switch ClusterProvider { + case "eks": + err = DownloadFilesFromS3(testDataS3Bucket, appDir, downloadDir, appFileList) + case "azure": + containerName := "/" + azureDataContainer + "/" + appDir + err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDir, containerName, appFileList) + case "gcp": + err = DownloadFilesFromGCP(testDataS3Bucket, appDir, downloadDir, appFileList) + } + + if err != nil { + return fmt.Errorf("unable to download %s app files: %w", version, err) + } + return nil +} + +// UploadAppFiles uploads app files to the appropriate cloud provider and returns the uploaded file paths. +func UploadAppFiles(ctx context.Context, testcaseEnvInst *TestCaseEnv, testS3Bucket, testDir, downloadDir string, appFileList []string, version string) ([]string, error) { + var uploadedFiles []string + var err error + + switch ClusterProvider { + case "eks": + testcaseEnvInst.Log.Info("Uploading apps to S3", "version", version) + uploadedFiles, err = UploadFilesToS3(testS3Bucket, testDir, appFileList, downloadDir) + case "azure": + testcaseEnvInst.Log.Info("Uploading apps to Azure", "version", version) + uploadedFiles, err = UploadFilesToAzure(ctx, StorageAccount, StorageAccountKey, downloadDir, testDir, appFileList) + case "gcp": + testcaseEnvInst.Log.Info("Uploading apps to GCP", "version", version) + uploadedFiles, err = UploadFilesToGCP(testS3Bucket, testDir, appFileList, downloadDir) + } + + if err != nil { + return nil, fmt.Errorf("unable to upload %s apps: %w", version, err) + } + return uploadedFiles, nil +} + +// DeleteUploadedFiles removes previously uploaded app files from the appropriate cloud provider. +func DeleteUploadedFiles(ctx context.Context, testS3Bucket string, uploadedApps []string) { + switch ClusterProvider { + case "eks": + DeleteFilesOnS3(testS3Bucket, uploadedApps) + case "azure": + azureBlobClient := &AzureBlobClient{} + azureBlobClient.DeleteFilesOnAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, uploadedApps) + case "gcp": + DeleteFilesOnGCP(testS3Bucket, uploadedApps) + } +} diff --git a/test/testenv/mcutil.go b/test/testenv/mcutil.go index 36250ae96..f5d61c997 100644 --- a/test/testenv/mcutil.go +++ b/test/testenv/mcutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,13 +17,15 @@ package testenv import ( "context" - "encoding/json" "fmt" "os/exec" "strings" + "time" + enterprisev4 "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/pkg/splunk/enterprise" corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -52,30 +54,7 @@ func getMCSts(ns string) string { return strings.Split(string(output), "\n")[1] } -// CheckMCPodReady check if monitoring pod is ready. Checking status of MC pod and Stateful set. -func CheckMCPodReady(ns string) bool { - // Check Status of monitoring console statefulset - stsLine := getMCSts(ns) - if len(stsLine) == 0 { - return false - } - stsSlice := strings.Fields(stsLine) - logf.Log.Info("MC statefulset found", "POD", stsSlice[0], "READY", stsSlice[1]) - stsReady := strings.Contains(stsSlice[1], "1/1") - - // Check Status of monitoring console pod - podLine := getMCPod(ns) - if len(podLine) == 0 { - return false - } - podSlice := strings.Fields(podLine) - logf.Log.Info("MC Pod Found", "POD", podSlice[0], "READY", podSlice[1]) - podReady := strings.Contains(podSlice[1], "1/1") && strings.Contains(podSlice[2], "Running") - - return stsReady && podReady -} - -// GetConfiguredPeers get list of Peers Configured on Montioring Console +// GetConfiguredPeers get list of Peers Configured on Monitoring Console func GetConfiguredPeers(ns string, mcName string) []string { podName := fmt.Sprintf(MonitoringConsolePod, mcName) var peerList []string @@ -100,22 +79,10 @@ func GetConfiguredPeers(ns string, mcName string) []string { } } } - logf.Log.Info("Peer List found on MC Pod", "MC POD", podName, "Configured Peers", peerList) + logf.Log.Info("Peer List found on MC Pod", "mcPod", podName, "configuredPeers", peerList) return peerList } -// DeleteMCPod delete monitoring console deployment -func DeleteMCPod(ns string) { - mcSts := fmt.Sprintf(MonitoringConsoleSts, ns) - output, err := exec.Command("kubectl", "delete", "sts", "-n", ns, mcSts).Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete sts -n %s %s", ns, mcSts) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("Monitoring Console Stateful Set deleted", "Statefulset", mcSts, "stdout", output) - } -} - // CheckPodNameOnMC Check given pod is configured on Monitoring console pod func CheckPodNameOnMC(ns string, mcName string, podName string) bool { // Get Peers configured on Monitoring Console @@ -124,7 +91,7 @@ func CheckPodNameOnMC(ns string, mcName string, podName string) bool { found := false for _, peer := range peerList { if strings.Contains(peer, podName) { - logf.Log.Info("Check Peer matches on pod", "Pod Name", podName, "Peer in peer list", peer) + logf.Log.Info("Check Peer matches on pod", "podName", podName, "peerInPeerList", peer) found = true break } @@ -134,19 +101,12 @@ func CheckPodNameOnMC(ns string, mcName string, podName string) bool { // GetPodIP returns IP address of a POD as a string func GetPodIP(ns string, podName string) string { - output, err := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() + podDetails, err := getPodDetails(ns, podName) if err != nil { - cmd := fmt.Sprintf("kubectl get pods -n %s %s -o json", ns, podName) - logf.Log.Error(err, "Failed to execute command", "command", cmd) + logf.Log.Error(err, "Failed to get pod details", "pod", podName) return "" } - restResponse := PodDetailsStruct{} - err = json.Unmarshal([]byte(output), &restResponse) - if err != nil { - logf.Log.Error(err, "Failed to parse cluster searchheads") - return "" - } - return restResponse.Status.PodIP + return podDetails.Status.PodIP } // GetMCConfigMap gets config map for give Monitoring Console Name @@ -157,12 +117,76 @@ func GetMCConfigMap(ctx context.Context, deployment *Deployment, ns string, mcNa logf.Log.Error(err, "Failed to get Monitoring Console Config Map") return mcConfigMap, err } - logf.Log.Info("MC Config Map contents", "MC CONFIG MAP NAME", mcConfigMapName, "Data", mcConfigMap.Data) + logf.Log.Info("MC Config Map contents", "mcConfigMapName", mcConfigMapName, "data", mcConfigMap.Data) return mcConfigMap, err } // CheckPodNameInString checks for pod name in string func CheckPodNameInString(podName string, configString string) bool { - logf.Log.Info("Check MC Config String has Pod configured", "Monitoring Console Config Map Pod Config String", configString, "POD String", podName) + logf.Log.Info("Check MC Config String has Pod configured", "configString", configString, "podName", podName) return strings.Contains(configString, podName) } + +// MCReconfigParams holds the service name and URL parameters that differ between +// V3 (master) and V4 (manager) Monitoring Console tests. +type MCReconfigParams struct { + CMServiceNameFmt string // format string for CM service name (e.g., ClusterMasterServiceName) + CMURLKey string // config map URL key (e.g., "SPLUNK_CLUSTER_MASTER_URL" or splcommon.ClusterManagerURL) +} + +// MCVersionConfig captures the API-version-specific behaviour that differs +// between V3 (master) and V4 (manager) monitoring console tests. +type MCVersionConfig struct { + MCReconfigParams + + NamePrefix string + Label string + + // DeployC3WithMC deploys a C3 single-site cluster with the given MC ref. + DeployC3WithMC func(ctx context.Context, d *Deployment, name string, replicas int, shc bool, mcRef string) error + + // DeployM4WithMC deploys an M4 multisite cluster with the given MC ref. + DeployM4WithMC func(ctx context.Context, d *Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error + + // NewCMObject returns a new, empty cluster-coordinator CR + // (*ClusterMaster for V3, *ClusterManager for V4). + NewCMObject func() client.Object + + // VerifyCMReady asserts the cluster coordinator has reached Ready phase. + VerifyCMReady func(ctx context.Context, d *Deployment, te *TestCaseEnv) error + + // SHCReconfigTimeout is the timeout used when verifying MC config strings + // after an SHC MC-ref reconfig (0 means use the synchronous check). + SHCReconfigTimeout time.Duration + + // VerifyMCTwoReadyAfterSHC controls whether MC Two is explicitly + // verified ready after the SHC reconfig step. + VerifyMCTwoReadyAfterSHC bool +} + +// ReconfigCMWithNewMC updates the Cluster Manager's MC ref to a new Monitoring Console, +// verifies the CM is ready, and deploys the new MC. +func ReconfigCMWithNewMC(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, cfg MCVersionConfig) (string, *enterprisev4.MonitoringConsole, error) { + mcTwoName := deployment.GetName() + "-two" + cm := cfg.NewCMObject() + if err := testcaseEnvInst.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, cm, deployment.GetName(), mcTwoName); err != nil { + return "", nil, fmt.Errorf("unable to update CM MC ref: %w", err) + } + if err := cfg.VerifyCMReady(ctx, deployment, testcaseEnvInst); err != nil { + return "", nil, fmt.Errorf("cluster manager not ready after MC reconfig: %w", err) + } + mcTwo, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + if err != nil { + return "", nil, fmt.Errorf("unable to deploy Monitoring Console Two: %w", err) + } + return mcTwoName, mcTwo, nil +} + +// DeployMCAndVerifyRFSF deploys a Monitoring Console and verifies RF/SF is met. +func DeployMCAndVerifyRFSF(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, mcRef string) error { + _, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) + if err != nil { + return err + } + return testcaseEnvInst.VerifyRFSFMet(ctx, deployment) +} diff --git a/test/testenv/multisite_helpers.go b/test/testenv/multisite_helpers.go new file mode 100644 index 000000000..bd762e819 --- /dev/null +++ b/test/testenv/multisite_helpers.go @@ -0,0 +1,89 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" +) + +// VerifyIndexOnAllSites verifies that an index exists on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnAllSites(ctx context.Context, deployment *Deployment, siteCount int, indexName string) error { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + if err := testcaseenv.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName); err != nil { + return err + } + } + return nil +} + +// IngestDataOnAllSites ingests data to an index on all indexer pods across all sites +func IngestDataOnAllSites(ctx context.Context, deployment *Deployment, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, deployment, logFile, indexName, podName) + } +} + +// RollHotToWarmOnAllSites rolls hot buckets to warm on all indexer pods across all sites +func RollHotToWarmOnAllSites(ctx context.Context, deployment *Deployment, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + RollHotToWarm(ctx, deployment, podName, indexName) + } +} + +// VerifyIndexOnS3AllSites verifies that an index exists on S3 for all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnS3AllSites(ctx context.Context, deployment *Deployment, siteCount int, indexName string) error { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + if err := testcaseenv.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName); err != nil { + return err + } + } + return nil +} + +// VerifyCPULimitsOnAllSites verifies CPU limits on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyCPULimitsOnAllSites(deployment *Deployment, siteCount int, expectedCPULimit string) error { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + if err := testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit); err != nil { + return err + } + } + return nil +} + +// MultisiteIndexerWorkflow encapsulates the common workflow for multisite indexer operations: +// verify index, ingest data, roll to warm, verify on S3 +func (testcaseenv *TestCaseEnv) MultisiteIndexerWorkflow(ctx context.Context, deployment *Deployment, siteCount int, indexName string) error { + // Verify index exists on all sites + if err := testcaseenv.VerifyIndexOnAllSites(ctx, deployment, siteCount, indexName); err != nil { + return err + } + + // Ingest data on all sites + IngestDataOnAllSites(ctx, deployment, siteCount, indexName) + + // Roll hot to warm on all sites + RollHotToWarmOnAllSites(ctx, deployment, siteCount, indexName) + + // Verify index on S3 for all sites + return testcaseenv.VerifyIndexOnS3AllSites(ctx, deployment, siteCount, indexName) +} diff --git a/test/testenv/remote_index_utils.go b/test/testenv/remote_index_utils.go index 551a377d0..a557324e4 100644 --- a/test/testenv/remote_index_utils.go +++ b/test/testenv/remote_index_utils.go @@ -3,6 +3,7 @@ package testenv import ( "context" "encoding/json" + "os" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -60,19 +61,6 @@ func GetIndexOnPod(ctx context.Context, deployment *Deployment, podName string, return indexFound, indexData } -// RestartSplunk Restart splunk inside the container -func RestartSplunk(ctx context.Context, deployment *Deployment, podName string) bool { - stdin := "/opt/splunk/bin/splunk restart -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - return true -} - // RollHotToWarm rolls hot buckets to warm for a given index and pod func RollHotToWarm(ctx context.Context, deployment *Deployment, podName string, indexName string) bool { stdin := "/opt/splunk/bin/splunk _internal call /data/indexes/" + indexName + "/roll-hot-buckets admin:$(cat /mnt/splunk-secrets/password)" @@ -130,6 +118,27 @@ func GenerateIndexVolumeSpecAzureManagedID(volumeName string, endpoint string, p } } +// GenerateVolumeSpecForProvider returns a VolumeSpec slice appropriate for the +// current ClusterProvider (eks, azure, gcp). For Azure it respects the +// AZURE_MANAGED_ID_ENABLED environment variable. +func (testenvInstance *TestCaseEnv) GenerateVolumeSpecForProvider(ctx context.Context, volumeName string) []enterpriseApi.VolumeSpec { + secretName := testenvInstance.GetIndexSecretName() + switch ClusterProvider { + case "eks": + return []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetS3Endpoint(), secretName, "aws", "s3", GetDefaultS3Region())} + case "azure": + if os.Getenv("AZURE_MANAGED_ID_ENABLED") == "false" { + return []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzure(volumeName, GetAzureEndpoint(ctx), secretName, "azure", "blob")} + } + return []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob")} + case "gcp": + return []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), secretName, "gcp", "gcs", GetDefaultS3Region())} + default: + testenvInstance.Log.Info("Failed to identify provider: Should be 'eks' or 'azure' or 'gcp'") + return nil + } +} + // GenerateIndexSpec return VolumeSpec struct with given values func GenerateIndexSpec(indexName string, volName string) enterpriseApi.IndexSpec { return enterpriseApi.IndexSpec{ diff --git a/test/testenv/s3utils.go b/test/testenv/s3utils.go index 445e395f1..220ce24a1 100644 --- a/test/testenv/s3utils.go +++ b/test/testenv/s3utils.go @@ -32,11 +32,6 @@ var ( s3deleteWaitTime = 60 ) -// GetSmartStoreIndexesBucet returns smartstore test bucket name -func GetSmartStoreIndexesBucet() string { - return testIndexesS3Bucket -} - // GetDefaultS3Region returns default AWS Region func GetDefaultS3Region() string { return s3Region @@ -53,9 +48,9 @@ func CheckPrefixExistsOnS3(prefix string) bool { resp := GetFileListOnS3(dataBucket, prefix) for _, key := range resp { - logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) + logf.Log.Info("Checking key", "key", *key.Key) if strings.Contains(*key.Key, prefix) { - logf.Log.Info("Prefix found on bucket", "Prefix", prefix, "KEY", *key.Key) + logf.Log.Info("Prefix found on bucket", "prefix", prefix, "key", *key.Key) return true } } @@ -94,7 +89,7 @@ func DownloadFileFromS3(dataBucket string, filename string, s3FilePath string, d // Create empty file on OS File System file, err := os.Create(filepath.Join(downloadDir, filename)) if err != nil { - logf.Log.Error(err, "Failed to create file", "Filename", file) + logf.Log.Error(err, "Failed to create file", "filename", file) } defer file.Close() @@ -112,7 +107,7 @@ func DownloadFileFromS3(dataBucket string, filename string, s3FilePath string, d }) if err != nil { - logf.Log.Error(err, "Failed to download file", "Bucket", dataBucket, "Path", s3FilePath, "Filename", filename) + logf.Log.Error(err, "Failed to download file", "bucket", dataBucket, "path", s3FilePath, "filename", filename) return "", err } @@ -174,7 +169,7 @@ func DeleteFileOnS3(bucket string, filename string) error { svc := s3.NewFromConfig(*cfg) _, err = svc.DeleteObject(context.TODO(), &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(filename)}) if err != nil { - logf.Log.Error(err, "Unable to delete object from bucket", "Object Name", filename, "Bucket Name", bucket) + logf.Log.Error(err, "Unable to delete object from bucket", "objectName", filename, "bucketName", bucket) } waiter := s3.NewObjectNotExistsWaiter(svc) @@ -182,7 +177,7 @@ func DeleteFileOnS3(bucket string, filename string) error { Bucket: aws.String(bucket), Key: aws.String(filename), }, time.Duration(time.Duration(s3deleteWaitTime).Seconds())) - logf.Log.Info("Deleted file on S3", "File Name", filename, "Bucket", bucket) + logf.Log.Info("Deleted file on S3", "fileName", filename, "bucket", bucket) return err } @@ -191,12 +186,12 @@ func GetFilesInPathOnS3(bucket string, path string) []string { resp := GetFileListOnS3(bucket, path) var files []string for _, key := range resp { - logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) + logf.Log.Info("Checking key", "key", *key.Key) if strings.Contains(*key.Key, path) { filename := strings.Replace(*key.Key, path, "", -1) // This condition filters out directories as GetFileListOnS3 returns files and directories with their absolute path's if len(filename) > 1 { - logf.Log.Info("File found on bucket", "Prefix", path, "KEY", *key.Key) + logf.Log.Info("File found on bucket", "prefix", path, "key", *key.Key) files = append(files, filename) } } @@ -207,10 +202,10 @@ func GetFilesInPathOnS3(bucket string, path string) []string { // DownloadFilesFromS3 download given list of files from S3 to the given directory func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading file from S3", "File name", key) + logf.Log.Info("Downloading file from S3", "fileName", key) _, err := DownloadFileFromS3(testDataS3Bucket, key, s3AppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -221,19 +216,19 @@ func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir s func UploadFilesToS3(testS3Bucket string, s3TestDir string, applist []string, downloadDir string) ([]string, error) { var uploadedFiles []string for _, key := range applist { - logf.Log.Info("Uploading file to S3", "File name", key) + logf.Log.Info("Uploading file to S3", "fileName", key) fileLocation := filepath.Join(downloadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open file", "File name", key) + logf.Log.Error(err, "Unable to open file", "fileName", key) return nil, err } fileName, err := UploadFileToS3(testS3Bucket, key, s3TestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } - logf.Log.Info("File upload to test S3", "File name", fileName) + logf.Log.Info("File upload to test S3", "fileName", fileName) uploadedFiles = append(uploadedFiles, fileName) } return uploadedFiles, nil diff --git a/test/testenv/search_head_cluster_utils.go b/test/testenv/search_head_cluster_utils.go index 012064501..4dd326676 100644 --- a/test/testenv/search_head_cluster_utils.go +++ b/test/testenv/search_head_cluster_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,39 +18,11 @@ package testenv import ( "context" "fmt" - "os/exec" "strings" logf "sigs.k8s.io/controller-runtime/pkg/log" ) -// DeleteSHC delete Search Head Cluster in given namespace -func DeleteSHC(ns string) { - output, err := exec.Command("kubectl", "delete", "shc", "-n", ns, "--all").Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete shc -n %s --all", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("SHC deleted", "Namespace", ns, "stdout", output) - } -} - -// SHCInNamespace returns true if SHC is present in namespace -func SHCInNamespace(ns string) bool { - output, err := exec.Command("kubectl", "get", "searchheadcluster", "-n", ns).Output() - deleted := true - if err != nil { - cmd := fmt.Sprintf("kubectl get shc -n %s", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - return deleted - } - logf.Log.Info("Output of command", "Output", string(output)) - if strings.Contains(string(output), "No resources found in default namespace") { - deleted = false - } - return deleted -} - // DeployerAppChecksum Get the checksum for each app on the deployer func DeployerAppChecksum(ctx context.Context, deployment *Deployment) map[string]string { appChecksum := make(map[string]string) @@ -143,7 +115,7 @@ func DeployerBundlePushstatus(ctx context.Context, deployment *Deployment, ns st } for appName := range appChecksum { if _, present := appBundlePush[appName]; !present { - logf.Log.Info("Deployer app not found on any members", "Appname", appName) + logf.Log.Info("Deployer app not found on any members", "appName", appName) return make(map[string]int) } } diff --git a/test/testenv/search_utils.go b/test/testenv/search_utils.go index b0e85cc66..3df735bfd 100644 --- a/test/testenv/search_utils.go +++ b/test/testenv/search_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,107 +60,64 @@ type SearchJobResponseResults struct { SplunkServer string `json:"splunk_server"` } -// PerformSearchSync performs a syncronous search within splunk and returns the search results -func PerformSearchSync(ctx context.Context, podName string, search string, deployment *Deployment) (string, error) { - // Build the search curl command and send it to an instance - curlCmd := "curl -ks -u" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - url := "https://localhost:8089/services/search/jobs/export" - - searchReq := fmt.Sprintf("%s %s:%s %s -d output_mode=json -d search=\"search %s\"", curlCmd, username, password, url, search) - command := []string{"/bin/sh"} - searchReqResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, searchReq, false) - _ = stderr +// splunkdCurlExec builds and executes a curl command against the local splunkd REST API on the given pod. +func splunkdCurlExec(ctx context.Context, deployment *Deployment, podName string, curlArgs string) (string, error) { + stdin := fmt.Sprintf("curl -ks -u admin:$(cat /mnt/splunk-secrets/password) %s", curlArgs) + stdout, _, err := deployment.PodExecCommand(ctx, podName, []string{"/bin/sh"}, stdin, false) if err != nil { - logf.Log.Error(err, "Failed to execute cmd on pod", "pod", podName, "command", command) + logf.Log.Error(err, "Failed to execute curl on pod", "pod", podName) return "", err } + return stdout, nil +} - logf.Log.Info("Output of search Query", "Search", search, "Output", searchReqResp) - - // Since results can have multiple formats depending on the search SPL, leave this response as a string - return searchReqResp, err +// PerformSearchSync performs a syncronous search within splunk and returns the search results +func PerformSearchSync(ctx context.Context, deployment *Deployment, podName string, search string) (string, error) { + curlArgs := fmt.Sprintf("https://localhost:8089/services/search/jobs/export -d output_mode=json -d search=\"search %s\"", search) + resp, err := splunkdCurlExec(ctx, deployment, podName, curlArgs) + if err != nil { + return "", err + } + logf.Log.Info("Output of search Query", "search", search, "output", resp) + return resp, nil } // PerformSearchReq makes a search request for a search to be performed. Returns a sid to be used to check for status and results -func PerformSearchReq(ctx context.Context, podName string, search string, deployment *Deployment) (string, error) { - // Build the search curl command - curlCmd := "curl -ks -u" - url := "https://localhost:8089/services/search/jobs" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - - searchReq := fmt.Sprintf("%s %s:%s %s -d output_mode=json -d search=\"search %s\"", curlCmd, username, password, url, search) - - // Send search request to instance - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, searchReq, false) - _ = stderr +func PerformSearchReq(ctx context.Context, deployment *Deployment, podName string, search string) (string, error) { + curlArgs := fmt.Sprintf("https://localhost:8089/services/search/jobs -d output_mode=json -d search=\"search %s\"", search) + stdout, err := splunkdCurlExec(ctx, deployment, podName, curlArgs) if err != nil { - logf.Log.Error(err, "Failed to execute cmd on pod", "pod", podName, "command", command) return "", err } + logf.Log.Info("Output of search Query", "search", search, "output", stdout) - logf.Log.Info("Output of search Query", "Search", search, "Output", stdout) - - // Get SID var searchReqResult map[string]interface{} - jsonErr := json.Unmarshal([]byte(stdout), &searchReqResult) - if jsonErr != nil { - logf.Log.Error(jsonErr, "Failed to unmarshal JSON Search Request Response to get SID") - return "", jsonErr + if err := json.Unmarshal([]byte(stdout), &searchReqResult); err != nil { + logf.Log.Error(err, "Failed to unmarshal JSON search request response") + return "", err } sid := searchReqResult["sid"].(string) - return sid, err + return sid, nil } // GetSearchStatus checks the search status for a given -func GetSearchStatus(ctx context.Context, podName string, sid string, deployment *Deployment) (*SearchJobStatusResponse, error) { - // Build search status request curl command - curlCmd := "curl -ks -u" - url := "https://localhost:8089/services/search/jobs" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - - searchStatusReq := fmt.Sprintf("%s %s:%s %s/%s -d output_mode=json", curlCmd, username, password, url, sid) - - // Send search status request to instance - command := []string{"/bin/sh"} - searchStatusResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, searchStatusReq, false) +func GetSearchStatus(ctx context.Context, deployment *Deployment, podName string, sid string) (*SearchJobStatusResponse, error) { + curlArgs := fmt.Sprintf("https://localhost:8089/services/search/jobs/%s -d output_mode=json", sid) + resp, err := splunkdCurlExec(ctx, deployment, podName, curlArgs) if err != nil { - logf.Log.Error(err, "Failed to execute cmd on pod", "pod", podName, "command", command, "stderr", stderr) return nil, err } - // Parse resulting JSON - searchStatusResult := SearchJobStatusResponse{} - jsonErr := json.Unmarshal([]byte(searchStatusResp), &searchStatusResult) - if jsonErr != nil { - logf.Log.Error(jsonErr, "Failed to unmarshal JSON Search Status Response to get SID") - return nil, jsonErr + var result SearchJobStatusResponse + if err := json.Unmarshal([]byte(resp), &result); err != nil { + logf.Log.Error(err, "Failed to unmarshal JSON search status response") + return nil, err } - return &searchStatusResult, err + return &result, nil } // GetSearchResults retrieve the results for a given once the search status isDone == true -func GetSearchResults(ctx context.Context, podName string, sid string, deployment *Deployment) (string, error) { - // Build search results request curl command - curlCmd := "curl -ks -u" - url := "https://localhost:8089/services/search/jobs" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - - searchResultsReq := fmt.Sprintf("%s %s:%s %s/%s/results/ --get -d output_mode=json", curlCmd, username, password, url, sid) - - // Send search results request to instance - command := []string{"/bin/sh"} - searchResultsResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, searchResultsReq, false) - if err != nil { - logf.Log.Error(err, "Failed to execute cmd on pod", "pod", podName, "command", command, "stderr", stderr) - return "", err - } - - // Since results can have multiple formats depending on the search SPL (transforming vs. non-transforming, etc.), leave this response as a string - return searchResultsResp, err +func GetSearchResults(ctx context.Context, deployment *Deployment, podName string, sid string) (string, error) { + curlArgs := fmt.Sprintf("https://localhost:8089/services/search/jobs/%s/results/ --get -d output_mode=json", sid) + return splunkdCurlExec(ctx, deployment, podName, curlArgs) } diff --git a/test/testenv/secretutil.go b/test/testenv/secretutil.go index 2bd513468..c6dd8aacc 100644 --- a/test/testenv/secretutil.go +++ b/test/testenv/secretutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,14 +46,14 @@ func GetSecretStruct(ctx context.Context, deployment *Deployment, ns string, sec secretObject := &corev1.Secret{} err := deployment.GetInstance(ctx, secretName, secretObject) if err != nil { - deployment.testenv.Log.Error(err, "Unable to get secret object", "Secret Name", secretName, "Namespace", ns) + deployment.testenv.Log.Error(err, "Unable to get secret object", "secretName", secretName, "namespace", ns) } return secretObject, err } // ModifySecretObject Modifies the secret object with given data func ModifySecretObject(ctx context.Context, deployment *Deployment, ns string, secretName string, data map[string][]byte) error { - logf.Log.Info("Modify secret object", "Secret Name", secretName, "Data", data) + logf.Log.Info("Modify secret object", "secretName", secretName, "data", data) secret := newSecretSpec(ns, secretName, data) err := deployment.UpdateCR(ctx, secret) @@ -66,7 +66,7 @@ func ModifySecretObject(ctx context.Context, deployment *Deployment, ns string, // DeleteSecretObject Deletes the entire secret object func DeleteSecretObject(ctx context.Context, deployment *Deployment, ns string, secretName string) error { - logf.Log.Info("Delete secret object", "Secret Name", secretName, "Namespace", ns) + logf.Log.Info("Delete secret object", "secretName", secretName, "namespace", ns) secret := newSecretSpec(ns, secretName, map[string][]byte{}) err := deployment.DeleteCR(ctx, secret) if err != nil { @@ -85,12 +85,12 @@ func GetMountedKey(ctx context.Context, deployment *Deployment, podName string, logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "" } - logf.Log.Info("Key found on pod", "Pod Name", podName, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Key found on pod", "podName", podName, "stdout", stdout, "stderr", stderr) return stdout } -// GetRandomeHECToken generates a random HEC token -func GetRandomeHECToken() string { +// GetRandomHECToken generates a random HEC token +func GetRandomHECToken() string { return fmt.Sprintf("%s-%s-%s-%s-%s", RandomHex(8), RandomHex(4), RandomHex(4), RandomHex(4), RandomHex(12)) } @@ -99,18 +99,18 @@ func GetSecretFromServerConf(ctx context.Context, deployment *Deployment, podNam filePath := "/opt/splunk/etc/system/local/server.conf" confline, err := GetConfLineFromPod(ctx, podName, filePath, ns, configName, stanza, true) if err != nil { - logf.Log.Error(err, "Failed to get secret from pod", "Pod Name", podName, "Secret Name", configName) + logf.Log.Error(err, "Failed to get secret from pod", "podName", podName, "secretName", configName) return "", "", err } secretList := strings.Split(confline, "=") key := strings.TrimSpace(secretList[0]) - value := DecryptSplunkEncodedSecret(ctx, deployment, podName, ns, strings.TrimSpace(secretList[1])) + value := DecryptSplunkEncodedSecret(ctx, deployment, podName, strings.TrimSpace(secretList[1])) return key, value, nil } // DecryptSplunkEncodedSecret Decrypt Splunk Secret like pass4SymmKey On Given Pod -func DecryptSplunkEncodedSecret(ctx context.Context, deployment *Deployment, podName string, ns string, secretValue string) string { +func DecryptSplunkEncodedSecret(ctx context.Context, deployment *Deployment, podName string, secretValue string) string { stdin := fmt.Sprintf("/opt/splunk/bin/splunk show-decrypted --value '%s'", secretValue) command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) @@ -120,7 +120,7 @@ func DecryptSplunkEncodedSecret(ctx context.Context, deployment *Deployment, pod } logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - logf.Log.Info("Decrypted Key Value", "Decrypted Key", stdout) + logf.Log.Info("Decrypted Key Value", "decryptedKey", stdout) return strings.TrimSuffix(stdout, "\n") } @@ -157,7 +157,7 @@ func GetVersionedSecretNames(ns string, version int) []string { } } } - logf.Log.Info("Versioned Secret Objects Found in Namespace", "NameSpace", ns, "Versioned Secrets", splunkSecrets) + logf.Log.Info("Versioned Secret Objects Found in Namespace", "namespace", ns, "versionedSecrets", splunkSecrets) return splunkSecrets } @@ -198,7 +198,7 @@ func GetSecretFromInputsConf(ctx context.Context, deployment *Deployment, podNam filePath := "/opt/splunk/etc/apps/splunk_httpinput/local/inputs.conf" confline, err := GetConfLineFromPod(ctx, podName, filePath, ns, configName, stanza, true) if err != nil { - logf.Log.Error(err, "Failed to get secret from pod", "Pod Name", podName, "Secret Name", configName) + logf.Log.Error(err, "Failed to get secret from pod", "podName", podName, "secretName", configName) return "", "", err } secretList := strings.Split(confline, "=") @@ -206,3 +206,15 @@ func GetSecretFromInputsConf(ctx context.Context, deployment *Deployment, podNam value := strings.TrimSpace(secretList[1]) return key, value, nil } + +// GenerateAndApplySecretUpdate creates randomized secret data and applies it to the namespace-scoped +// secret object, returning the updated data map for subsequent verification. +func GenerateAndApplySecretUpdate(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, namespaceScopedSecretName string) (map[string][]byte, error) { + modifiedHecToken := GetRandomHECToken() + modifiedValue := RandomDNSName(24) + updatedSecretData := GetSecretDataMap(modifiedHecToken, modifiedValue, modifiedValue, modifiedValue, modifiedValue) + if err := ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData); err != nil { + return nil, fmt.Errorf("unable to update secret object: %w", err) + } + return updatedSecretData, nil +} diff --git a/test/testenv/test_setup_helpers.go b/test/testenv/test_setup_helpers.go new file mode 100644 index 000000000..ba9e516d5 --- /dev/null +++ b/test/testenv/test_setup_helpers.go @@ -0,0 +1,297 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/joho/godotenv" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" +) + +// SetupOption configures optional parameters for SetupTestCaseEnv. +type SetupOption func(*setupOptions) + +type setupOptions struct { + timeout *time.Duration +} + +// WithTimeout overrides the default test timeout for the deployment. +func WithTimeout(seconds int) SetupOption { + return func(o *setupOptions) { + d := time.Duration(seconds) * time.Second + o.timeout = &d + } +} + +// SetupTestCaseEnv creates a new test case environment and deployment for use in BeforeEach blocks. +// It also validates test prerequisites immediately to fail fast before any long operations. +func SetupTestCaseEnv(testenvInstance *TestEnv, namePrefix string, opts ...SetupOption) (*TestCaseEnv, *Deployment, error) { + var o setupOptions + for _, opt := range opts { + opt(&o) + } + + name := fmt.Sprintf("%s-%s", namePrefix+testenvInstance.GetName(), RandomDNSName(3)) + testcaseEnvInst, err := NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) + if err != nil { + return nil, nil, fmt.Errorf("unable to create testcaseenv: %w", err) + } + + deployment, err := testcaseEnvInst.NewDeployment(RandomDNSName(3), o.timeout) + if err != nil { + return nil, nil, fmt.Errorf("unable to create deployment: %w", err) + } + + if err = testcaseEnvInst.ValidateTestPrerequisites(context.TODO(), deployment); err != nil { + return nil, nil, fmt.Errorf("test prerequisites validation failed: %w", err) + } + + return testcaseEnvInst, deployment, nil +} + +// TeardownTestCaseEnv handles the common teardown logic for test case environments. +func TeardownTestCaseEnv(testcaseEnvInst *TestCaseEnv, deployment *Deployment) error { + if types.SpecState(ginkgo.CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil { + testcaseEnvInst.SkipTeardown = true + } + } + + if deployment != nil { + deployment.Teardown() + } + + if testcaseEnvInst != nil { + if err := testcaseEnvInst.Teardown(); err != nil { + return fmt.Errorf("teardown failed: %w", err) + } + } + return nil +} + +// CleanupOperatorFile deletes the test_file.img from the operator pod's app download directory +// if filePresentOnOperator is true. +func CleanupOperatorFile(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, filePresentOnOperator bool) { + if filePresentOnOperator { + opPod := testcaseEnvInst.GetOperatorPodName() + podDownloadPath := filepath.Join(AppDownloadVolume, "test_file.img") + DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) + } +} + +// TeardownAppFrameworkTestCaseEnv handles teardown for app framework tests with provider-specific +// cloud storage cleanup. cloudCleanup is called only if SkipTeardown is false. +func TeardownAppFrameworkTestCaseEnv(ctx context.Context, testcaseEnvInst *TestCaseEnv, deployment *Deployment, cloudCleanup func(), filePresentOnOperator bool) error { + if err := TeardownTestCaseEnv(testcaseEnvInst, deployment); err != nil { + return err + } + + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown && cloudCleanup != nil { + cloudCleanup() + } + + CleanupOperatorFile(ctx, deployment, testcaseEnvInst, filePresentOnOperator) + return nil +} + +// S3CloudCleanup returns a cleanup function that deletes the given files from an S3 bucket. +func S3CloudCleanup(bucket string, uploadedApps []string) func() { + return func() { + DeleteFilesOnS3(bucket, uploadedApps) + } +} + +// AzureCloudCleanup returns a cleanup function that deletes the given files from Azure Blob storage. +func AzureCloudCleanup(ctx context.Context, uploadedApps []string) func() { + return func() { + azureBlobClient := &AzureBlobClient{} + azureBlobClient.DeleteFilesOnAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, uploadedApps) + } +} + +// GCPCloudCleanup returns a cleanup function that deletes the given files from a GCP bucket. +func GCPCloudCleanup(bucket string, uploadedApps []string) func() { + return func() { + DeleteFilesOnGCP(bucket, uploadedApps) + } +} + +// LoadEnvFile traverses up the directory tree from the current working directory +// to find and load a .env file using godotenv. Returns nil if no .env file is found. +func LoadEnvFile() error { + dir, err := os.Getwd() + if err != nil { + return err + } + + for { + envFile := filepath.Join(dir, ".env") + if _, err := os.Stat(envFile); err == nil { + return godotenv.Load(envFile) + } + + parentDir := filepath.Dir(dir) + if parentDir == dir { + return nil + } + dir = parentDir + } +} + +// SetupS3AppsSuite initialises the test environment and, when running on EKS, +// downloads the V1 and V2 app sets from S3. +func SetupS3AppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string, error) { + testenvInst, err := NewDefaultTestEnv(suiteName) + if err != nil { + return nil, nil, nil, fmt.Errorf("unable to create test env: %w", err) + } + + if ClusterProvider == "eks" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + if err = DownloadFilesFromS3(testDataBucket, appDirV1, downloadDirV1, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V1 app files: %w", err) + } + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + if err = DownloadFilesFromS3(testDataBucket, appDirV2, downloadDirV2, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V2 app files: %w", err) + } + + return testenvInst, appListV1, appListV2, nil + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil, nil +} + +// CleanupLocalAppDownloads tears down the test environment and removes locally +// downloaded app directories after a suite run. +func CleanupLocalAppDownloads(testenvInst *TestEnv, dirs ...string) error { + if testenvInst != nil { + if err := testenvInst.Teardown(); err != nil { + return fmt.Errorf("teardown failed: %w", err) + } + } + for _, dir := range dirs { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("unable to delete locally downloaded app files from %s: %w", dir, err) + } + } + return nil +} + +// SetupAzureAppsSuite initialises the test environment and, when running on Azure, +// downloads the V1 and V2 app sets from Azure Blob. +func SetupAzureAppsSuite(suiteName, downloadDirV1, downloadDirV2 string) (*TestEnv, []string, []string, error) { + testenvInst, err := NewDefaultTestEnv(suiteName) + if err != nil { + return nil, nil, nil, fmt.Errorf("unable to create test env: %w", err) + } + + if ClusterProvider == "azure" { + ctx := context.TODO() + + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + containerName := "/test-data/appframework/v1apps/" + if err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV1, containerName, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V1 app files: %w", err) + } + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + containerName = "/test-data/appframework/v2apps/" + if err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV2, containerName, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V2 app files: %w", err) + } + + return testenvInst, appListV1, appListV2, nil + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil, nil +} + +// SetupGCPAppsSuite initialises the test environment and, when running on GCP, +// downloads the V1 and V2 app sets from GCS. +func SetupGCPAppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string, error) { + testenvInst, err := NewDefaultTestEnv(suiteName) + if err != nil { + return nil, nil, nil, fmt.Errorf("unable to create test env: %w", err) + } + + if ClusterProvider == "gcp" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + testenvInst.Log.Info("logging download details", "bucket", testDataBucket, "appDirV1", appDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) + if err = DownloadFilesFromGCP(testDataBucket, appDirV1, downloadDirV1, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V1 app files: %w", err) + } + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + if err = DownloadFilesFromGCP(testDataBucket, appDirV2, downloadDirV2, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V2 app files: %w", err) + } + + return testenvInst, appListV1, appListV2, nil + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil, nil +} + +// SetupLicenseConfigMap downloads the license file from the appropriate provider +// and creates a license config map. +func SetupLicenseConfigMap(ctx context.Context, testcaseEnvInst *TestCaseEnv) error { + downloadDir := "licenseFolder" + var licenseFilePath string + var err error + + switch ClusterProvider { + case "eks": + licenseFilePath, err = DownloadLicenseFromS3Bucket() + case "azure": + licenseFilePath, err = DownloadLicenseFromAzure(ctx, downloadDir) + case "gcp": + licenseFilePath, err = DownloadLicenseFromGCPBucket() + default: + testcaseEnvInst.Log.Info("Skipping license download", "provider", ClusterProvider) + return nil + } + + if err != nil { + return fmt.Errorf("unable to download license file: %w", err) + } + + if err := testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath); err != nil { + return fmt.Errorf("unable to create license config map: %w", err) + } + return nil +} diff --git a/test/testenv/test_workflows.go b/test/testenv/test_workflows.go new file mode 100644 index 000000000..d2ec14235 --- /dev/null +++ b/test/testenv/test_workflows.go @@ -0,0 +1,153 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package testenv + +import ( + "context" + "fmt" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// WorkflowResult contains the result of a workflow execution. +// Only fields that are actually populated by the workflow are kept here; +// extend the struct when a new workflow needs to return additional CRs. +type WorkflowResult struct { + Standalone *enterpriseApi.Standalone +} + +// RunStandaloneDeploymentWorkflow deploys a Standalone instance and verifies it's ready +func (testcaseEnvInst *TestCaseEnv) RunStandaloneDeploymentWorkflow(ctx context.Context, deployment *Deployment) (*WorkflowResult, error) { + standalone, err := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, "", "") + if err != nil { + return nil, err + } + return &WorkflowResult{Standalone: standalone}, nil +} + +// RunC3DeploymentWorkflow deploys a C3 cluster (CM + IDXC + SHC) and verifies all components are ready +func (testcaseEnvInst *TestCaseEnv) RunC3DeploymentWorkflow(ctx context.Context, deployment *Deployment, indexerReplicas int, mcRef string) (*WorkflowResult, error) { + if err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, true, mcRef); err != nil { + return nil, fmt.Errorf("unable to deploy C3 cluster: %w", err) + } + + if err := testcaseEnvInst.VerifyClusterReadyAndRFSF(ctx, deployment); err != nil { + return nil, fmt.Errorf("cluster not ready: %w", err) + } + + return &WorkflowResult{}, nil +} + +// RunM4DeploymentWorkflow deploys a M4 multisite cluster and verifies all components are ready +func (testcaseEnvInst *TestCaseEnv) RunM4DeploymentWorkflow(ctx context.Context, deployment *Deployment, indexerReplicas int, siteCount int, mcRef string) (*WorkflowResult, error) { + if err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), indexerReplicas, siteCount, mcRef); err != nil { + return nil, fmt.Errorf("unable to deploy M4 cluster: %w", err) + } + + if err := testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady); err != nil { + return nil, fmt.Errorf("M4 cluster not ready: %w", err) + } + if err := testcaseEnvInst.VerifyRFSFMet(ctx, deployment); err != nil { + return nil, fmt.Errorf("RF/SF not met: %w", err) + } + + return &WorkflowResult{}, nil +} + +// RunM1DeploymentWorkflow deploys an M1 multisite Indexer Cluster (no SHC) and verifies components +func (testcaseEnvInst *TestCaseEnv) RunM1DeploymentWorkflow(ctx context.Context, deployment *Deployment, indexerReplicas int, siteCount int) (*WorkflowResult, error) { + if err := deployment.DeployMultisiteCluster(ctx, deployment.GetName(), indexerReplicas, siteCount, ""); err != nil { + return nil, fmt.Errorf("unable to deploy M1 cluster: %w", err) + } + + if err := testcaseEnvInst.VerifyM1ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady); err != nil { + return nil, fmt.Errorf("M1 cluster not ready: %w", err) + } + if err := testcaseEnvInst.VerifyRFSFMet(ctx, deployment); err != nil { + return nil, fmt.Errorf("RF/SF not met: %w", err) + } + + return &WorkflowResult{}, nil +} + +// RunStandaloneWithServiceAccountWorkflow deploys a Standalone with a service account +func (testcaseEnvInst *TestCaseEnv) RunStandaloneWithServiceAccountWorkflow(ctx context.Context, deployment *Deployment, serviceAccountName string) (*WorkflowResult, error) { + if err := testcaseEnvInst.CreateServiceAccount(serviceAccountName); err != nil { + return nil, fmt.Errorf("unable to create service account: %w", err) + } + + name := deployment.GetName() + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + ServiceAccount: serviceAccountName, + }, + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, name, spec) + if err != nil { + return nil, fmt.Errorf("unable to deploy standalone with service account: %w", err) + } + + if err = testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, name, standalone); err != nil { + return nil, fmt.Errorf("standalone not ready: %w", err) + } + + standalonePodName := fmt.Sprintf(StandalonePod, name, 0) + if err = testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(ctx, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName); err != nil { + return nil, fmt.Errorf("service account not configured: %w", err) + } + + return &WorkflowResult{Standalone: standalone}, nil +} + +// RunDeleteStandaloneWorkflow deploys and deletes a standalone instance +func (testcaseEnvInst *TestCaseEnv) RunDeleteStandaloneWorkflow(ctx context.Context, deployment *Deployment) error { + result, err := testcaseEnvInst.RunStandaloneDeploymentWorkflow(ctx, deployment) + if err != nil { + return fmt.Errorf("unable to deploy Standalone instance: %w", err) + } + + if err := deployment.DeleteCR(ctx, result.Standalone); err != nil { + return fmt.Errorf("unable to delete Standalone instance: %w", err) + } + return nil +} + +// RunDeleteC3Workflow deploys and deletes a C3 cluster +func (testcaseEnvInst *TestCaseEnv) RunDeleteC3Workflow(ctx context.Context, deployment *Deployment, indexerReplicas int) error { + if _, err := testcaseEnvInst.RunC3DeploymentWorkflow(ctx, deployment, indexerReplicas, ""); err != nil { + return err + } + + name := deployment.GetName() + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.IndexerCluster{}, name+"-idxc"); err != nil { + return fmt.Errorf("unable to delete Indexer Cluster: %w", err) + } + + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.SearchHeadCluster{}, name+"-shc"); err != nil { + return fmt.Errorf("unable to delete Search Head Cluster: %w", err) + } + + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.ClusterManager{}, name); err != nil { + return fmt.Errorf("unable to delete Cluster Manager: %w", err) + } + + return nil +} diff --git a/test/testenv/testcaseenv.go b/test/testenv/testcaseenv.go index 78c7d6a00..9807b951c 100644 --- a/test/testenv/testcaseenv.go +++ b/test/testenv/testcaseenv.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -116,11 +116,6 @@ func (testenv *TestCaseEnv) GetSplunkImage() string { return testenv.splunkImage } -// IsOperatorInstalledClusterWide returns if operator is installed clusterwide -func (testenv *TestCaseEnv) IsOperatorInstalledClusterWide() string { - return testenv.clusterWideOperator -} - func (testenv *TestCaseEnv) setup() error { testenv.Log.Info("testenv initializing.\n") @@ -361,7 +356,7 @@ func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { operator := &appsv1.Deployment{} err = testenv.GetKubeClient().Get(ctx, namespacedName, operator) if err != nil { - testenv.Log.Error(err, "Unable to get operator", "operator name", testenv.operatorName) + testenv.Log.Error(err, "Unable to get operator", "operatorName", testenv.operatorName) return err } @@ -386,7 +381,7 @@ func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { // update the operator deployment now err = testenv.GetKubeClient().Update(ctx, operator) if err != nil { - testenv.Log.Error(err, "Unable to update operator", "operator name", testenv.operatorName) + testenv.Log.Error(err, "Unable to update operator", "operatorName", testenv.operatorName) return err } @@ -652,12 +647,18 @@ func (testenv *TestCaseEnv) GetLMConfigMap() string { return testenv.licenseCMName } -// NewDeployment creates a new deployment -func (testenv *TestCaseEnv) NewDeployment(name string) (*Deployment, error) { +// NewDeployment creates a new deployment. If timeout is non-nil it overrides +// the default SpecifiedTestTimeout. +func (testenv *TestCaseEnv) NewDeployment(name string, timeout *time.Duration) (*Deployment, error) { + t := time.Duration(SpecifiedTestTimeout) * time.Second + if timeout != nil { + t = *timeout + } + d := Deployment{ name: testenv.GetName() + "-" + name, testenv: testenv, - testTimeout: time.Duration(SpecifiedTestTimeout) * time.Second, + testTimeout: t, } return &d, nil diff --git a/test/testenv/testenv.go b/test/testenv/testenv.go index dc72345cf..addb4c7c7 100644 --- a/test/testenv/testenv.go +++ b/test/testenv/testenv.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -69,10 +69,10 @@ const ( // StandalonePod Template String for standalone pod StandalonePod = "splunk-%s-standalone-%d" - // LicenseManagerPod Template String for standalone pod + // LicenseManagerPod Template String for License Manager pod LicenseManagerPod = "splunk-%s-license-manager-%d" - // LicenseMasterPod Template String for standalone pod + // LicenseMasterPod Template String for License Master pod LicenseMasterPod = "splunk-%s-" + splcommon.LicenseManager + "-%d" // IngestorPod Template String for ingestor pod @@ -122,11 +122,17 @@ const ( // ClusterMasterServiceName Cluster Master Service Template String ClusterMasterServiceName = "splunk-%s-cluster-master-service" - // DeployerServiceName Cluster Manager Service Template String + // DeployerServiceName Deployer Service Template String DeployerServiceName = "splunk-%s-shc-deployer-service" // CRUpdateRetryCount if CR Update fails retry these many time CRUpdateRetryCount = 10 + + // LogLineCount is the default number of log lines to ingest for test data + LogLineCount = 2000 + + // DefaultIngestIndex is the default index name used for test data ingestion + DefaultIngestIndex = "main" ) var ( diff --git a/test/testenv/timeouts.go b/test/testenv/timeouts.go index 36b69d70b..0737de5b2 100644 --- a/test/testenv/timeouts.go +++ b/test/testenv/timeouts.go @@ -55,6 +55,12 @@ const DefaultTimeout = 15 * time.Minute // C3 deployments require bundle push across all indexers and SHC deployer which can exceed 5 minutes. const AppInstallTimeout = 10 * time.Minute +// AppStateVerificationTimeout is the timeout for VerifyAppState polls that +// try to catch a transient app-framework phase (e.g. download-in-progress). +// M4 clusters need time to initialise before app processing begins, so this +// value is generous. +const AppStateVerificationTimeout = 60 * time.Minute + // SetupTeardownTimeout limits BeforeEach setup and AfterEach teardown nodes. // Prevents hung setup or cleanup from consuming the entire suite timeout. const SetupTeardownTimeout = 10 * time.Minute diff --git a/test/testenv/util.go b/test/testenv/util.go index 8ef0c6a4e..888299cd9 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,21 +17,16 @@ package testenv import ( "context" - "encoding/json" "fmt" "math/rand" "os" "os/exec" "path" - "reflect" - "sort" "strings" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/v2" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -752,7 +747,7 @@ func newStandaloneWithSpec(name, ns string, spec enterpriseApi.StandaloneSpec) * return &new } -// newMonitoringConsoleSpec returns MC Spec with given name, namespace and license manager Ref +// newMonitoringConsoleSpec returns MC Spec with given name, namespace and License Manager Ref func newMonitoringConsoleSpec(name, ns, LicenseManagerRef, splunkImage string) *enterpriseApi.MonitoringConsole { licenseMasterRef, licenseManagerRef := swapLicenseManager(name, LicenseManagerRef) @@ -821,66 +816,8 @@ func DumpGetPods(ns string) []string { return splunkPods } -// DumpDescribePods prints and returns list of pods in the namespace -func DumpDescribePods(ns string) []string { - output, err := exec.Command("kubectl", "describe", "pods", "-n", ns).Output() - var splunkPods []string - if err != nil { - //cmd := fmt.Sprintf("kubectl get pods -n %s", ns) - //logf.Log.Error(err, "Failed to execute command", "command", cmd) - return nil - } - for _, line := range strings.Split(string(output), "\n") { - logf.Log.Info(line) - if strings.HasPrefix(line, "splunk") && !strings.HasPrefix(line, "splunk-op") { - splunkPods = append(splunkPods, strings.Fields(line)[0]) - } - } - return splunkPods -} - -// DumpGetTopNodes prints and returns Node load information -func DumpGetTopNodes() []string { - output, err := exec.Command("kubectl", "top", "nodes").Output() - var splunkNodes []string - if err != nil { - //cmd := "kubectl top nodes" - //logf.Log.Error(err, "Failed to execute command", "command", cmd) - return nil - } - if len(output) > 0 { - for _, line := range strings.Split(string(output), "\n") { - if len(line) > 0 { - logf.Log.Info(line) - splunkNodes = append(splunkNodes, strings.Fields(line)[0]) - } - } - } - return splunkNodes -} - -// DumpGetTopPods prints and returns Node load information -func DumpGetTopPods(ns string) []string { - output, err := exec.Command("kubectl", "top", "pods", "-n", ns).Output() - var splunkPods []string - if err != nil { - //cmd := fmt.Sprintf("kubectl top pods -n %s", ns) - //logf.Log.Error(err, "Failed to execute command", "command", cmd) - return nil - } - if len(output) > 0 { - for _, line := range strings.Split(string(output), "\n") { - if len(line) > 0 { - logf.Log.Info(line) - splunkPods = append(splunkPods, strings.Fields(line)[0]) - } - } - } - return splunkPods -} - // GetOperatorPodName returns name of operator pod in the namespace -func GetOperatorPodName(testcaseEnvInst *TestCaseEnv) string { +func (testcaseEnvInst *TestCaseEnv) GetOperatorPodName() string { var ns string if testcaseEnvInst.clusterWideOperator != "true" { ns = testcaseEnvInst.GetName() @@ -904,7 +841,7 @@ func GetOperatorPodName(testcaseEnvInst *TestCaseEnv) string { return splunkPods } } - logf.Log.Info("Operator pod is set to ", "operatorPod", splunkPods) + logf.Log.Info("Operator pod is set to", "operatorPod", splunkPods) return splunkPods } @@ -929,11 +866,8 @@ func DumpGetPvcs(ns string) []string { // GetConfLineFromPod gets given config from file on POD func GetConfLineFromPod(ctx context.Context, podName string, filePath string, ns string, configName string, stanza string, checkStanza bool) (string, error) { var config string - var err error - output, err := exec.CommandContext(ctx, "kubectl", "exec", "-n", ns, podName, "--", "cat", filePath).Output() + fileContent, err := GetConfFile(podName, filePath, ns) if err != nil { - cmd := fmt.Sprintf("kubectl exec -n %s %s -- cat %s", ns, podName, filePath) - logf.Log.Error(err, "Failed to execute command", "command", cmd) return config, err } @@ -943,7 +877,7 @@ func GetConfLineFromPod(ctx context.Context, podName string, filePath string, ns stanzaFound = false stanzaString = fmt.Sprintf("[%s]", stanza) } - for _, line := range strings.Split(string(output), "\n") { + for _, line := range strings.Split(fileContent, "\n") { // Check for empty lines to prevent an error in logic below if len(line) == 0 { continue @@ -955,7 +889,7 @@ func GetConfLineFromPod(ctx context.Context, podName string, filePath string, ns } continue } else if strings.HasPrefix(line, configName) { - logf.Log.Info(fmt.Sprintf("Configuration %s found at line %s", configName, line)) + logf.Log.Info("Configuration found", "configName", configName, "line", line) config = line break } @@ -974,7 +908,7 @@ func ExecuteCommandOnPod(ctx context.Context, deployment *Deployment, podName st logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } @@ -986,7 +920,7 @@ func ExecuteCommandOnOperatorPod(ctx context.Context, deployment *Deployment, po logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "shell", command, "command", stdin, "error", err.Error()) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } @@ -995,7 +929,7 @@ func GetConfigMap(ctx context.Context, deployment *Deployment, ns string, config configMap := &corev1.ConfigMap{} err := deployment.GetInstance(ctx, configMapName, configMap) if err != nil { - deployment.testenv.Log.Error(err, "Unable to get config map", "Config Map Name", configMap, "Namespace", ns) + deployment.testenv.Log.Error(err, "Unable to get config map", "configMapName", configMap, "namespace", ns) } return configMap, err } @@ -1066,7 +1000,7 @@ func newLicenseManagerWithGivenSpec(name, ns string, spec enterpriseApi.LicenseM return &new } -// newLicenseMasterWithGivenSpec create and initializes CR for License Manager Kind with Given Spec +// newLicenseMasterWithGivenSpec create and initializes CR for License Master Kind with Given Spec func newLicenseMasterWithGivenSpec(name, ns string, spec enterpriseApiV3.LicenseMasterSpec) *enterpriseApiV3.LicenseMaster { new := enterpriseApiV3.LicenseMaster{ TypeMeta: metav1.TypeMeta{ @@ -1124,19 +1058,9 @@ func GetDirsOrFilesInPath(ctx context.Context, deployment *Deployment, podName s return strings.Fields(stdout), err } -// CompareStringSlices checks if two string slices are matching -func CompareStringSlices(stringOne []string, stringTwo []string) bool { - if len(stringOne) != len(stringTwo) { - return false - } - sort.Strings(stringOne) - sort.Strings(stringTwo) - return reflect.DeepEqual(stringOne, stringTwo) -} - // CheckStringInSlice check if string is present in a slice func CheckStringInSlice(stringSlice []string, compString string) bool { - logf.Log.Info("Checking for string in slice", "String", compString, "String Slice", stringSlice) + logf.Log.Info("Checking for string in slice", "string", compString, "stringSlice", stringSlice) for _, item := range stringSlice { if strings.Contains(item, compString) { return true @@ -1168,29 +1092,18 @@ func GetPodUIDs(ns string) map[string]string { splunkPods := DumpGetPods(ns) for _, podName := range splunkPods { - output, _ := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() - restResponse := PodDetailsStruct{} - err := json.Unmarshal([]byte(output), &restResponse) + podDetails, err := getPodDetails(ns, podName) if err != nil { - logf.Log.Error(err, "Failed to parse splunk pods") + logf.Log.Error(err, "Failed to get pod details", "pod", podName) + continue } - splunkPodUIDs[podName] = restResponse.Metadata.UID + splunkPodUIDs[podName] = podDetails.Metadata.UID } return splunkPodUIDs } -// DeletePod Delete pod in the namespace -func DeletePod(ns string, podName string) error { - _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() - if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) - return err - } - return nil -} - // DeleteOperatorPod Delete Operator Pod in the namespace -func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { +func (testcaseEnvInst *TestCaseEnv) DeleteOperatorPod() error { var podName string var ns string if testcaseEnvInst.clusterWideOperator != "true" { @@ -1198,11 +1111,11 @@ func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { } else { ns = "splunk-operator" } - podName = GetOperatorPodName(testcaseEnvInst) + podName = testcaseEnvInst.GetOperatorPodName() _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) + logf.Log.Error(err, "Failed to delete operator pod ", "podName", podName, "namespace", ns) return err } return nil @@ -1214,7 +1127,7 @@ func DeleteFilesOnOperatorPod(ctx context.Context, deployment *Deployment, podNa cmd := fmt.Sprintf("rm -f %s", filepath) _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to delete file on pod ", "PodName", podName, "location", filepath, "command", cmd) + logf.Log.Error(err, "Failed to delete file on pod ", "podName", podName, "location", filepath, "command", cmd) return err } } @@ -1229,9 +1142,9 @@ func DumpGetSplunkVersion(ctx context.Context, ns string, deployment *Deployment if strings.Contains(podName, filterString) { stdout, err := ExecuteCommandOnPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to get splunkd version on the pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get splunkd version on the pod", "podName", podName) } - logf.Log.Info("Splunk Version Found", "Pod Name", podName, "Version", string(stdout)) + logf.Log.Info("Splunk Version Found", "podName", podName, "version", string(stdout)) } } } @@ -1241,7 +1154,7 @@ func CreateDummyFileOnOperator(ctx context.Context, deployment *Deployment, podN cmd := fmt.Sprintf("cd %s && dd if=/dev/zero of=./%s bs=4k iflag=fullblock,count_bytes count=%s", filepath, filename, size) _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to create file on the pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to create file on the pod", "podName", podName) return err } return nil @@ -1249,10 +1162,10 @@ func CreateDummyFileOnOperator(ctx context.Context, deployment *Deployment, podN // DeleteConfigMap Delete configMap in the namespace func DeleteConfigMap(ns string, ConfigMapName string) error { - logf.Log.Info("Delete configMap", "configMap Name", ConfigMapName) + logf.Log.Info("Delete configMap", "configMapName", ConfigMapName) _, err := exec.Command("kubectl", "delete", "configmap", "-n", ns, ConfigMapName).Output() if err != nil { - logf.Log.Error(err, "Failed to delete config Map", "ConfigMap Name", ConfigMapName, "Namespace", ns) + logf.Log.Error(err, "Failed to delete config Map", "configMapName", ConfigMapName, "namespace", ns) return err } return nil @@ -1288,16 +1201,17 @@ func GetAWSEnv(podName, ns string) (string, error) { return string(output), err } -func ValidateContent(confFileContent string, listOfStringsForValidation []string, shouldContain bool) { +func ValidateContent(confFileContent string, listOfStringsForValidation []string, shouldContain bool) error { for _, str := range listOfStringsForValidation { if shouldContain { if !strings.Contains(confFileContent, str) { - Expect(confFileContent).To(ContainSubstring(str), "Failed to find string "+str+" in conf file") + return fmt.Errorf("failed to find string %q in conf file content", str) } } else { if strings.Contains(confFileContent, str) { - Expect(confFileContent).ToNot(ContainSubstring(str), "Found string "+str+" in conf file, but it should not be there") + return fmt.Errorf("found string %q in conf file, but it should not be there", str) } } } + return nil } diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index ca515be09..cb365af41 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,12 +22,12 @@ import ( "fmt" "math/rand" "os/exec" + "reflect" "strings" "time" "sigs.k8s.io/controller-runtime/pkg/client" - gomega "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" wait "k8s.io/apimachinery/pkg/util/wait" @@ -76,163 +76,264 @@ type PodDetailsStruct struct { } `json:"status"` } -// VerifyMonitoringConsoleReady verify Monitoring Console CR is in Ready Status and does not flip-flop -func (testenv *TestCaseEnv) VerifyMonitoringConsoleReady(ctx context.Context, deployment *Deployment, mcName string, monitoringConsole *enterpriseApi.MonitoringConsole) { - // Use optimized watch to wait for Ready phase - err := testenv.WatchForMonitoringConsolePhase(ctx, deployment, testenv.GetName(), mcName, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "MonitoringConsole failed to reach Ready phase") +// getPodDetails fetches and unmarshals the JSON details for a single pod. +func getPodDetails(ns, podName string) (*PodDetailsStruct, error) { + output, err := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() + if err != nil { + return nil, fmt.Errorf("kubectl get pod %s in ns %s: %w", podName, ns, err) + } + var details PodDetailsStruct + if err := json.Unmarshal(output, &details); err != nil { + return nil, fmt.Errorf("unmarshal pod %s details: %w", podName, err) + } + return &details, nil +} - // Refresh the instance to get latest state - err = deployment.GetInstance(ctx, mcName, monitoringConsole) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("MonitoringConsole reached Ready phase", "instance", monitoringConsole.ObjectMeta.Name, "Phase", monitoringConsole.Status.Phase) - DumpGetPods(testenv.GetName()) +// PollConsistently verifies a condition holds for the entire duration. +// condFn should return nil if the condition holds, or an error if it fails. +// The check is abandoned early if ctx is cancelled. +func PollConsistently(ctx context.Context, duration, interval time.Duration, condFn func() error) error { + deadline := time.Now().Add(duration) + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if err := condFn(); err != nil { + return fmt.Errorf("consistency check failed: %w", err) + } + time.Sleep(interval) + } + return nil +} + +// VerifyMonitoringConsoleReady verify Monitoring Console CR is in Ready Status and does not flip-flop +func (testenv *TestCaseEnv) VerifyMonitoringConsoleReady(ctx context.Context, deployment *Deployment, mcName string, monitoringConsole *enterpriseApi.MonitoringConsole) error { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + err := deployment.GetInstance(ctx, mcName, monitoringConsole) + if err != nil { + return false, nil + } + testenv.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "phase", monitoringConsole.Status.Phase) + DumpGetPods(testenv.GetName()) + return monitoringConsole.Status.Phase == enterpriseApi.PhaseReady, nil + }) + if err != nil { + return fmt.Errorf("monitoring console %s failed to reach Ready phase: %w", mcName, err) + } // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, mcName, monitoringConsole) + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, mcName, monitoringConsole); err != nil { + testenv.Log.Info("Transient error refreshing MonitoringConsole during consistency check", "error", err) + } DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "monitoring-console") - return monitoringConsole.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if monitoringConsole.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("monitoring console phase flipped to %s", monitoringConsole.Status.Phase) + } + return nil + }) } // VerifyStandaloneReady verify Standalone is in ReadyStatus and does not flip-flop -func (testenv *TestCaseEnv) VerifyStandaloneReady(ctx context.Context, deployment *Deployment, deploymentName string, standalone *enterpriseApi.Standalone) { +func (testenv *TestCaseEnv) VerifyStandaloneReady(ctx context.Context, deployment *Deployment, deploymentName string, standalone *enterpriseApi.Standalone) error { // Use optimized watch to wait for Ready phase err := testenv.WatchForStandalonePhase(ctx, deployment, testenv.GetName(), standalone.Name, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "Standalone failed to reach Ready phase") + if err != nil { + return fmt.Errorf("standalone failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state err = deployment.GetInstance(ctx, standalone.Name, standalone) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("Standalone reached Ready phase", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) + if err != nil { + return fmt.Errorf("failed to get standalone instance: %w", err) + } + testenv.Log.Info("Standalone reached Ready phase", "instance", standalone.ObjectMeta.Name, "phase", standalone.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, standalone.Name, standalone) + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, standalone.Name, standalone); err != nil { + testenv.Log.Info("Transient error refreshing Standalone during consistency check", "error", err) + } DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "standalone") - return standalone.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if standalone.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("standalone phase flipped to %s", standalone.Status.Phase) + } + return nil + }) +} + +// VerifyStandalonePhaseAndReady verifies the Standalone reaches the given transitional phase +// (e.g. ScalingUp, Updating) and then returns to Ready without flip-flopping. +func (testenv *TestCaseEnv) VerifyStandalonePhaseAndReady(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase, standalone *enterpriseApi.Standalone) error { + if err := testenv.VerifyStandalonePhase(ctx, deployment, phase); err != nil { + return err + } + return testenv.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) } // VerifySearchHeadClusterReady verify SHC is in READY status and does not flip-flop -func (testenv *TestCaseEnv) VerifySearchHeadClusterReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifySearchHeadClusterReady(ctx context.Context, deployment *Deployment) error { instanceName := fmt.Sprintf("%s-shc", deployment.GetName()) // Use optimized watch to wait for Ready phase (checks both Phase and DeployerPhase) err := testenv.WatchForSearchHeadClusterPhase(ctx, deployment, testenv.GetName(), instanceName, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "SearchHeadCluster failed to reach Ready phase") + if err != nil { + return fmt.Errorf("SearchHeadCluster failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state shc := &enterpriseApi.SearchHeadCluster{} err = deployment.GetInstance(ctx, instanceName, shc) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("SearchHeadCluster reached Ready phase", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase, "DeployerPhase", shc.Status.DeployerPhase) + if err != nil { + return fmt.Errorf("failed to get SearchHeadCluster instance: %w", err) + } + testenv.Log.Info("SearchHeadCluster reached Ready phase", "instance", shc.ObjectMeta.Name, "phase", shc.Status.Phase, "deployerPhase", shc.Status.DeployerPhase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, deployment.GetName(), shc) - testenv.Log.Info("Check for Consistency Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, instanceName, shc); err != nil { + testenv.Log.Info("Transient error refreshing SearchHeadCluster during consistency check", "error", err) + } + testenv.Log.Info("Check for Consistency Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "phase", shc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-shc-") - return shc.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if shc.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("SHC phase flipped to %s", shc.Status.Phase) + } + return nil + }) } // VerifySingleSiteIndexersReady verify single site indexers go to ready state -func (testenv *TestCaseEnv) VerifySingleSiteIndexersReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifySingleSiteIndexersReady(ctx context.Context, deployment *Deployment) error { instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) // Use optimized watch to wait for Ready phase err := testenv.WatchForIndexerClusterPhase(ctx, deployment, testenv.GetName(), instanceName, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "IndexerCluster failed to reach Ready phase") + if err != nil { + return fmt.Errorf("IndexerCluster failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state idc := &enterpriseApi.IndexerCluster{} err = deployment.GetInstance(ctx, instanceName, idc) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("IndexerCluster reached Ready phase", "instance", instanceName, "Phase", idc.Status.Phase) + if err != nil { + return fmt.Errorf("failed to get IndexerCluster instance: %w", err) + } + testenv.Log.Info("IndexerCluster reached Ready phase", "instance", instanceName, "phase", idc.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, instanceName, idc) - testenv.Log.Info("Check for Consistency indexer instance's phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, instanceName, idc); err != nil { + testenv.Log.Info("Transient error refreshing IndexerCluster during consistency check", "error", err) + } + testenv.Log.Info("Check for Consistency indexer instance's phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-idxc-indexer-") - return idc.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if idc.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("indexer phase flipped to %s", idc.Status.Phase) + } + return nil + }) } // IngestorsReady verify ingestors go to ready state -func (testenv *TestCaseEnv) VerifyIngestorReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyIngestorReady(ctx context.Context, deployment *Deployment) error { instanceName := fmt.Sprintf("%s-ingest", deployment.GetName()) // Use optimized watch to wait for Ready phase err := testenv.WatchForIngestorClusterPhase(ctx, deployment, testenv.GetName(), instanceName, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "IngestorCluster failed to reach Ready phase") + if err != nil { + return fmt.Errorf("IngestorCluster failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state ingest := &enterpriseApi.IngestorCluster{} err = deployment.GetInstance(ctx, instanceName, ingest) - gomega.Expect(err).To(gomega.Succeed()) + if err != nil { + return fmt.Errorf("failed to get IngestorCluster instance: %w", err) + } testenv.Log.Info("IngestorCluster reached Ready phase", "instance", instanceName, "phase", ingest.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, instanceName, ingest) - + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, instanceName, ingest); err != nil { + testenv.Log.Info("Transient error refreshing IngestorCluster during consistency check", "error", err) + } testenv.Log.Info("Check for Consistency ingestor instance's phase to be ready", "instance", instanceName, "phase", ingest.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-ingest-") - - return ingest.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if ingest.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("ingestor phase flipped to %s", ingest.Status.Phase) + } + return nil + }) } // VerifyClusterManagerReady verify Cluster Manager Instance is in ready status -func (testenv *TestCaseEnv) VerifyClusterManagerReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyClusterManagerReady(ctx context.Context, deployment *Deployment) error { // Use optimized watch to wait for Ready phase err := testenv.WatchForClusterManagerPhase(ctx, deployment, testenv.GetName(), deployment.GetName(), enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "ClusterManager failed to reach Ready phase") + if err != nil { + return fmt.Errorf("ClusterManager failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state cm := &enterpriseApi.ClusterManager{} err = deployment.GetInstance(ctx, deployment.GetName(), cm) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("ClusterManager reached Ready phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + if err != nil { + return fmt.Errorf("failed to get ClusterManager instance: %w", err) + } + testenv.Log.Info("ClusterManager reached Ready phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, cluster-manager should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, deployment.GetName(), cm) - testenv.Log.Info("Check for Consistency "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, deployment.GetName(), cm); err != nil { + testenv.Log.Info("Transient error refreshing ClusterManager during consistency check", "error", err) + } + testenv.Log.Info("Check for Consistency "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "cluster-manager") - testenv.Log.Info("Check for Consistency cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) - return cm.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + testenv.Log.Info("Check for Consistency cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) + if cm.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("cluster manager phase flipped to %s", cm.Status.Phase) + } + return nil + }) } // VerifyClusterMasterReady verify Cluster Master Instance is in ready status -func (testenv *TestCaseEnv) VerifyClusterMasterReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyClusterMasterReady(ctx context.Context, deployment *Deployment) error { // Use optimized watch to wait for Ready phase err := testenv.WatchForClusterMasterPhase(ctx, deployment, testenv.GetName(), deployment.GetName(), enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "ClusterMaster failed to reach Ready phase") + if err != nil { + return fmt.Errorf("ClusterMaster failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state cm := &enterpriseApiV3.ClusterMaster{} err = deployment.GetInstance(ctx, deployment.GetName(), cm) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("ClusterMaster reached Ready phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + if err != nil { + return fmt.Errorf("failed to get ClusterMaster instance: %w", err) + } + testenv.Log.Info("ClusterMaster reached Ready phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, cluster-master should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, deployment.GetName(), cm) - testenv.Log.Info("Check for Consistency cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) - return cm.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, deployment.GetName(), cm); err != nil { + testenv.Log.Info("Transient error refreshing ClusterMaster during consistency check", "error", err) + } + testenv.Log.Info("Check for Consistency cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) + if cm.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("cluster master phase flipped to %s", cm.Status.Phase) + } + return nil + }) } // VerifyIndexersReady verify indexers of all sites go to ready state -func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment *Deployment, siteCount int) { +func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment *Deployment, siteCount int) error { siteIndexerMap := map[string][]string{} for site := 1; site <= siteCount; site++ { siteName := fmt.Sprintf("site%d", site) @@ -240,50 +341,56 @@ func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment siteIndexerMap[siteName] = []string{fmt.Sprintf("splunk-%s-indexer-0", instanceName)} // Ensure indexers go to Ready phase idc := &enterpriseApi.IndexerCluster{} - gomega.Eventually(func() enterpriseApi.Phase { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, instanceName, idc) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetPods(testenv.GetName()) - - return idc.Status.Phase - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return idc.Status.Phase == enterpriseApi.PhaseReady, nil + }) + if err != nil { + return fmt.Errorf("indexer site %s failed to reach Ready phase: %w", siteName, err) + } // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, instanceName, idc) - testenv.Log.Info("Check for Consistency indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + err = PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, instanceName, idc); err != nil { + testenv.Log.Info("Transient error refreshing IndexerCluster site during consistency check", "error", err) + } + testenv.Log.Info("Check for Consistency indexer site instance phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-idxc-indexer-") - return idc.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if idc.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("indexer phase flipped to %s for site %s", idc.Status.Phase, siteName) + } + return nil + }) + if err != nil { + return err + } } + return nil } -// VerifyIndexerClusterMultisiteStatus verify indexer Cluster is configured as multisite -func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Context, deployment *Deployment, siteCount int) { +// VerifyIndexerClusterMultisiteStatus verify Indexer Cluster is configured as multisite +func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Context, deployment *Deployment, siteCount int) error { siteIndexerMap := map[string][]string{} for site := 1; site <= siteCount; site++ { siteName := fmt.Sprintf("site%d", site) instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) siteIndexerMap[siteName] = []string{fmt.Sprintf("splunk-%s-indexer-0", instanceName)} } - gomega.Eventually(func() map[string][]string { - var podName string - if strings.Contains(deployment.name, "master") { - podName = fmt.Sprintf(ClusterMasterPod, deployment.GetName()) - } else { - podName = fmt.Sprintf(ClusterManagerPod, deployment.GetName()) - } + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + podName := GetCMPodName(deployment) stdin := "curl -ks -u admin:$(cat /mnt/splunk-secrets/password) https://localhost:8089/services/cluster/manager/sites?output_mode=json" command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) if err != nil { - testenv.Log.Error(err, "Failed to execute command", "on pod", podName, "command", command) - return map[string][]string{} + testenv.Log.Error(err, "Failed to execute command", "onPod", podName, "command", command) + return false, nil } - testenv.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + testenv.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) siteIndexerResponse := ClusterManagerSitesResponse{} json.Unmarshal([]byte(stdout), &siteIndexerResponse) siteIndexerStatus := map[string][]string{} @@ -293,140 +400,149 @@ func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Cont siteIndexerStatus[site.Name] = append(siteIndexerStatus[site.Name], peer.ServerName) } } - return siteIndexerStatus - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(siteIndexerMap)) + return reflect.DeepEqual(siteIndexerStatus, siteIndexerMap), nil + }) } -// VerifyRFSFMet verify RF SF is met on cluster manager -func (testenv *TestCaseEnv) VerifyRFSFMet(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() bool { +// VerifyRFSFMet verify RF SF is met on Cluster Manager +func (testenv *TestCaseEnv) VerifyRFSFMet(ctx context.Context, deployment *Deployment) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { rfSfStatus := CheckRFSF(ctx, deployment) - testenv.Log.Info("Verifying RF SF is met", "Status", rfSfStatus) - return rfSfStatus - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).Should(gomega.Equal(true)) + testenv.Log.Info("Verifying RF SF is met", "status", rfSfStatus) + return rfSfStatus, nil + }) } -// VerifyNoDisconnectedSHPresentOnCM is present on cluster manager -func (testenv *TestCaseEnv) VerifyNoDisconnectedSHPresentOnCM(ctx context.Context, deployment *Deployment) { - gomega.Consistently(func() bool { +// VerifyNoDisconnectedSHPresentOnCM verifies no disconnected SH is present on Cluster Manager +func (testenv *TestCaseEnv) VerifyNoDisconnectedSHPresentOnCM(ctx context.Context, deployment *Deployment) error { + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { shStatus := CheckSearchHeadRemoved(ctx, deployment) - testenv.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "Status", shStatus) - return shStatus - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) -} - -// VerifyNoSHCInNamespace verify no SHC is present in namespace -func (testenv *TestCaseEnv) VerifyNoSHCInNamespace(deployment *Deployment) { - gomega.Eventually(func() bool { - shcStatus := SHCInNamespace(testenv.GetName()) - testenv.Log.Info("Verifying no Search Head Cluster is present in namespace", "Status", shcStatus) - return shcStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(false)) + testenv.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "status", shStatus) + if !shStatus { + return fmt.Errorf("disconnected search head found on Cluster Manager") + } + return nil + }) } // VerifyLicenseManagerReady verify LM is in ready status and does not flip flop -func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deployment *Deployment) error { LicenseManager := &enterpriseApi.LicenseManager{} testenv.Log.Info("Verifying License Manager becomes READY") - gomega.Eventually(func() enterpriseApi.Phase { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, deployment.GetName(), LicenseManager) if err != nil { - return enterpriseApi.PhaseError + return false, nil } testenv.Log.Info("Waiting for License Manager instance status to be ready", - "instance", LicenseManager.ObjectMeta.Name, "Phase", LicenseManager.Status.Phase) + "instance", LicenseManager.ObjectMeta.Name, "phase", LicenseManager.Status.Phase) DumpGetPods(testenv.GetName()) - - return LicenseManager.Status.Phase - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return LicenseManager.Status.Phase == enterpriseApi.PhaseReady, nil + }) + if err != nil { + return fmt.Errorf("license manager failed to reach Ready phase: %w", err) + } // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, deployment.GetName(), LicenseManager) - return LicenseManager.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, deployment.GetName(), LicenseManager); err != nil { + testenv.Log.Info("Transient error refreshing LicenseManager during consistency check", "error", err) + } + if LicenseManager.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("license manager phase flipped to %s", LicenseManager.Status.Phase) + } + return nil + }) } // VerifyLicenseMasterReady verify LM is in ready status and does not flip flop -func (testenv *TestCaseEnv) VerifyLicenseMasterReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyLicenseMasterReady(ctx context.Context, deployment *Deployment) error { LicenseMaster := &enterpriseApiV3.LicenseMaster{} testenv.Log.Info("Verifying License Master becomes READY") - gomega.Eventually(func() enterpriseApi.Phase { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, deployment.GetName(), LicenseMaster) if err != nil { - return enterpriseApi.PhaseError + return false, nil } testenv.Log.Info("Waiting for License Master instance status to be ready", - "instance", LicenseMaster.ObjectMeta.Name, "Phase", LicenseMaster.Status.Phase) + "instance", LicenseMaster.ObjectMeta.Name, "phase", LicenseMaster.Status.Phase) DumpGetPods(testenv.GetName()) - - return LicenseMaster.Status.Phase - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return LicenseMaster.Status.Phase == enterpriseApi.PhaseReady, nil + }) + if err != nil { + return fmt.Errorf("license master failed to reach Ready phase: %w", err) + } // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { - _ = deployment.GetInstance(ctx, deployment.GetName(), LicenseMaster) - return LicenseMaster.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + if err := deployment.GetInstance(ctx, deployment.GetName(), LicenseMaster); err != nil { + testenv.Log.Info("Transient error refreshing LicenseMaster during consistency check", "error", err) + } + if LicenseMaster.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("license master phase flipped to %s", LicenseMaster.Status.Phase) + } + return nil + }) } // VerifyLMConfiguredOnPod verify LM is configured on given POD -func VerifyLMConfiguredOnPod(ctx context.Context, deployment *Deployment, podName string) { - gomega.Consistently(func() bool { +func VerifyLMConfiguredOnPod(ctx context.Context, deployment *Deployment, podName string) error { + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { lmConfigured := CheckLicenseManagerConfigured(ctx, deployment, podName) - return lmConfigured - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) + if !lmConfigured { + return fmt.Errorf("license manager not configured on pod %s", podName) + } + return nil + }) } // VerifyServiceAccountConfiguredOnPod check if given service account is configured on given pod -func (testenv *TestCaseEnv) VerifyServiceAccountConfiguredOnPod(deployment *Deployment, ns string, podName string, serviceAccount string) { - gomega.Consistently(func() bool { - output, err := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() +func (testenv *TestCaseEnv) VerifyServiceAccountConfiguredOnPod(ctx context.Context, ns string, podName string, serviceAccount string) error { + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { + podDetails, err := getPodDetails(ns, podName) if err != nil { - cmd := fmt.Sprintf("kubectl get pods -n %s %s -o json", ns, podName) - testenv.Log.Error(err, "Failed to execute command", "command", cmd) - return false + testenv.Log.Error(err, "Failed to get pod details", "pod", podName) + return err } - restResponse := PodDetailsStruct{} - err = json.Unmarshal([]byte(output), &restResponse) - if err != nil { - testenv.Log.Error(err, "Failed to parse cluster Search heads") - return false + testenv.Log.Info("Service Account on Pod", "found", podDetails.Spec.ServiceAccount, "expected", serviceAccount) + if !strings.Contains(serviceAccount, podDetails.Spec.ServiceAccount) { + return fmt.Errorf("service account mismatch on pod %s: expected %s, found %s", podName, serviceAccount, podDetails.Spec.ServiceAccount) } - testenv.Log.Info("Service Account on Pod", "found", restResponse.Spec.ServiceAccount, "expected", serviceAccount) - return strings.Contains(serviceAccount, restResponse.Spec.ServiceAccount) - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) + return nil + }) } // VerifyIndexFoundOnPod verify index found on a given POD -func (testenv *TestCaseEnv) VerifyIndexFoundOnPod(ctx context.Context, deployment *Deployment, podName string, indexName string) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyIndexFoundOnPod(ctx context.Context, deployment *Deployment, podName string, indexName string) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { indexFound, _ := GetIndexOnPod(ctx, deployment, podName, indexName) testenv.Log.Info("Checking status of index on pod", "podName", podName, "indexName", indexName, "status", indexFound) - return indexFound - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(true)) + return indexFound, nil + }) } // VerifyIndexConfigsMatch verify index specific config -func (testenv *TestCaseEnv) VerifyIndexConfigsMatch(ctx context.Context, deployment *Deployment, podName string, indexName string, maxGlobalDataSizeMB int, maxGlobalRawDataSizeMB int) { - gomega.Consistently(func() bool { +func (testenv *TestCaseEnv) VerifyIndexConfigsMatch(ctx context.Context, deployment *Deployment, podName string, indexName string, maxGlobalDataSizeMB int, maxGlobalRawDataSizeMB int) error { + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { indexFound, data := GetIndexOnPod(ctx, deployment, podName, indexName) testenv.Log.Info("Checking status of index on pod", "podName", podName, "indexName", indexName, "status", indexFound) - if indexFound { - if data.Content.MaxGlobalDataSizeMB == maxGlobalDataSizeMB && data.Content.MaxGlobalRawDataSizeMB == maxGlobalRawDataSizeMB { - testenv.Log.Info("Checking index configs", "maxGlobalDataSizeMB", data.Content.MaxGlobalDataSizeMB, "maxGlobalRawDataSizeMB", data.Content.MaxGlobalRawDataSizeMB) - return true - } + if !indexFound { + return fmt.Errorf("index %s not found on pod %s", indexName, podName) } - return false - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) + if data.Content.MaxGlobalDataSizeMB != maxGlobalDataSizeMB || data.Content.MaxGlobalRawDataSizeMB != maxGlobalRawDataSizeMB { + return fmt.Errorf("index config mismatch on pod %s: maxGlobalDataSizeMB=%d (expected %d), maxGlobalRawDataSizeMB=%d (expected %d)", + podName, data.Content.MaxGlobalDataSizeMB, maxGlobalDataSizeMB, data.Content.MaxGlobalRawDataSizeMB, maxGlobalRawDataSizeMB) + } + testenv.Log.Info("Checking index configs", "maxGlobalDataSizeMB", data.Content.MaxGlobalDataSizeMB, "maxGlobalRawDataSizeMB", data.Content.MaxGlobalRawDataSizeMB) + return nil + }) } // VerifyIndexExistsOnS3 Verify Index Exists on S3 -func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deployment *Deployment, indexName string, podName string) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deployment *Deployment, indexName string, podName string) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { indexFound := CheckPrefixExistsOnS3(indexName) testenv.Log.Info("Checking Index on S3", "indexName", indexName, "status", indexFound) // During testing found some false failure. Rolling index buckets again to ensure data is pushed to remote storage @@ -434,298 +550,181 @@ func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deploymen testenv.Log.Info("Index NOT found. Rolling buckets again", "indexName", indexName) RollHotToWarm(ctx, deployment, podName, indexName) } - return indexFound - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(true)) -} - -// VerifyRollingRestartFinished verify no rolling restart is active -func (testenv *TestCaseEnv) VerifyRollingRestartFinished(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() bool { - rollingRestartStatus := CheckRollingRestartStatus(ctx, deployment) - testenv.Log.Info("Rolling Restart Status", "active", rollingRestartStatus) - return rollingRestartStatus - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).Should(gomega.Equal(true)) + return indexFound, nil + }) } // VerifyConfOnPod Verify give conf and value on config file on pod -func (testenv *TestCaseEnv) VerifyConfOnPod(ctx context.Context, deployment *Deployment, podName string, confFilePath string, config string, value string) { - gomega.Consistently(func() bool { +func (testenv *TestCaseEnv) VerifyConfOnPod(ctx context.Context, podName string, confFilePath string, config string, value string) error { + return PollConsistently(ctx, ConsistentDuration, ConsistentPollInterval, func() error { confLine, err := GetConfLineFromPod(ctx, podName, confFilePath, testenv.GetName(), config, "", false) if err != nil { testenv.Log.Error(err, "Failed to get config on pod") - return false + return fmt.Errorf("failed to get config on pod %s: %w", podName, err) } if strings.Contains(confLine, config) && strings.Contains(confLine, value) { testenv.Log.Info("Config found", "config", config, "value", value, "confLine", confLine) - return true + return nil } testenv.Log.Info("Config NOT found") - return false - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) + return fmt.Errorf("config %s=%s not found on pod %s", config, value, podName) + }) } // VerifySearchHeadClusterPhase verify the phase of SHC matches given phase -func (testenv *TestCaseEnv) VerifySearchHeadClusterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifySearchHeadClusterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { shc := &enterpriseApi.SearchHeadCluster{} shcName := deployment.GetName() + "-shc" err := deployment.GetInstance(ctx, shcName, shc) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "Expected", phase, "Phase", shc.Status.Phase) + testenv.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "expected", phase, "phase", shc.Status.Phase) DumpGetPods(testenv.GetName()) - - return shc.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).Should(gomega.Equal(enterpriseApi.PhaseScalingUp)) + return shc.Status.Phase == phase, nil + }) } // VerifyIndexerClusterPhase verify the phase of idxc matches the given phase -func (testenv *TestCaseEnv) VerifyIndexerClusterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase, idxcName string) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifyIndexerClusterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase, idxcName string) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { idxc := &enterpriseApi.IndexerCluster{} err := deployment.GetInstance(ctx, idxcName, idxc) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "Expected", phase, "Phase", idxc.Status.Phase) + testenv.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "expected", phase, "phase", idxc.Status.Phase) DumpGetPods(testenv.GetName()) - - return idxc.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).Should(gomega.Equal(phase)) + return idxc.Status.Phase == phase, nil + }) } // VerifyStandalonePhase verify the phase of Standalone CR -func (testenv *TestCaseEnv) VerifyStandalonePhase(ctx context.Context, deployment *Deployment, crName string, phase enterpriseApi.Phase) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifyStandalonePhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { standalone := &enterpriseApi.Standalone{} err := deployment.GetInstance(ctx, deployment.GetName(), standalone) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "Expected", phase, " Actual Phase", standalone.Status.Phase) + testenv.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "expected", phase, "actualPhase", standalone.Status.Phase) DumpGetPods(testenv.GetName()) - - return standalone.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).Should(gomega.Equal(phase)) + return standalone.Status.Phase == phase, nil + }) } // VerifyMonitoringConsolePhase verify the phase of Monitoring Console CR -func (testenv *TestCaseEnv) VerifyMonitoringConsolePhase(ctx context.Context, deployment *Deployment, crName string, phase enterpriseApi.Phase) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifyMonitoringConsolePhase(ctx context.Context, deployment *Deployment, crName string, phase enterpriseApi.Phase) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { mc := &enterpriseApi.MonitoringConsole{} err := deployment.GetInstance(ctx, crName, mc) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "Expected", phase, " Actual Phase", mc.Status.Phase) + testenv.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "expected", phase, "actualPhase", mc.Status.Phase) DumpGetPods(testenv.GetName()) - - return mc.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).Should(gomega.Equal(phase)) + return mc.Status.Phase == phase, nil + }) } // GetResourceVersion get resource version id -func (testenv *TestCaseEnv) GetResourceVersion(ctx context.Context, deployment *Deployment, instance interface{}) string { - var newResourceVersion string - var err error - - switch cr := instance.(type) { - case *enterpriseApi.Standalone: - err = deployment.GetInstance(ctx, cr.Name, cr) - newResourceVersion = cr.ResourceVersion - case *enterpriseApi.LicenseManager: - err = deployment.GetInstance(ctx, cr.Name, cr) - newResourceVersion = cr.ResourceVersion - case *enterpriseApi.IndexerCluster: - err = deployment.GetInstance(ctx, cr.Name, cr) - newResourceVersion = cr.ResourceVersion - case *enterpriseApi.ClusterManager: - err = deployment.GetInstance(ctx, cr.Name, cr) - newResourceVersion = cr.ResourceVersion - case *enterpriseApi.MonitoringConsole: - err = deployment.GetInstance(ctx, cr.Name, cr) - newResourceVersion = cr.ResourceVersion - case *enterpriseApi.SearchHeadCluster: - err = deployment.GetInstance(ctx, cr.Name, cr) - newResourceVersion = cr.ResourceVersion - default: - return "-1" - } - if err != nil { +func (testenv *TestCaseEnv) GetResourceVersion(ctx context.Context, deployment *Deployment, obj client.Object) string { + if err := deployment.GetInstance(ctx, obj.GetName(), obj); err != nil { return "-1" } - return newResourceVersion + return obj.GetResourceVersion() } // VerifyCustomResourceVersionChanged verify the version id -func (testenv *TestCaseEnv) VerifyCustomResourceVersionChanged(ctx context.Context, deployment *Deployment, instance interface{}, resourceVersion string) { - var kind string - var newResourceVersion string - var name string - var err error - - gomega.Eventually(func() string { - switch cr := instance.(type) { - case *enterpriseApi.Standalone: - err = deployment.GetInstance(ctx, cr.Name, cr) - kind = cr.Kind - newResourceVersion = cr.ResourceVersion - name = cr.Name - case *enterpriseApi.LicenseManager: - err = deployment.GetInstance(ctx, cr.Name, cr) - kind = cr.Kind - newResourceVersion = cr.ResourceVersion - name = cr.Name - case *enterpriseApiV3.LicenseMaster: - err = deployment.GetInstance(ctx, cr.Name, cr) - kind = cr.Kind - newResourceVersion = cr.ResourceVersion - name = cr.Name - case *enterpriseApi.IndexerCluster: - err = deployment.GetInstance(ctx, cr.Name, cr) - kind = cr.Kind - newResourceVersion = cr.ResourceVersion - name = cr.Name - case *enterpriseApi.ClusterManager: - err = deployment.GetInstance(ctx, cr.Name, cr) - kind = cr.Kind - newResourceVersion = cr.ResourceVersion - name = cr.Name - case *enterpriseApiV3.ClusterMaster: - err = deployment.GetInstance(ctx, cr.Name, cr) - kind = cr.Kind - newResourceVersion = cr.ResourceVersion - name = cr.Name - case *enterpriseApi.MonitoringConsole: - err = deployment.GetInstance(ctx, cr.Name, cr) - kind = cr.Kind - newResourceVersion = cr.ResourceVersion - name = cr.Name - case *enterpriseApi.SearchHeadCluster: - err = deployment.GetInstance(ctx, cr.Name, cr) - newResourceVersion = cr.ResourceVersion - kind = cr.Kind - name = cr.Name - default: - return "-1" - } - if err != nil { - return "-1" +func (testenv *TestCaseEnv) VerifyCustomResourceVersionChanged(ctx context.Context, deployment *Deployment, obj client.Object, resourceVersion string) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + if err := deployment.GetInstance(ctx, obj.GetName(), obj); err != nil { + return false, nil } - testenv.Log.Info("Waiting for ", kind, " CR status", "instance", name, "Not Expected", resourceVersion, " Actual Resource Version", newResourceVersion) + newResourceVersion := obj.GetResourceVersion() + testenv.Log.Info("Waiting for CR status change", "type", fmt.Sprintf("%T", obj), "instance", obj.GetName(), "notExpected", resourceVersion, "actualResourceVersion", newResourceVersion) DumpGetPods(testenv.GetName()) - - return newResourceVersion - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).ShouldNot(gomega.Equal(resourceVersion)) + return newResourceVersion != resourceVersion, nil + }) } // VerifyCPULimits verifies value of CPU limits is as expected -func (testenv *TestCaseEnv) VerifyCPULimits(deployment *Deployment, podName string, expectedCPULimits string) { - gomega.Eventually(func() bool { - ns := testenv.GetName() - output, err := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() - if err != nil { - cmd := fmt.Sprintf("kubectl get pods -n %s %s -o json", ns, podName) - testenv.Log.Error(err, "Failed to execute command", "command", cmd) - return false - } - restResponse := PodDetailsStruct{} - err = json.Unmarshal([]byte(output), &restResponse) +func (testenv *TestCaseEnv) VerifyCPULimits(deployment *Deployment, podName string, expectedCPULimits string) error { + return wait.PollUntilContextTimeout(context.TODO(), PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + podDetails, err := getPodDetails(testenv.GetName(), podName) if err != nil { - testenv.Log.Error(err, "Failed to parse JSON") - return false + testenv.Log.Error(err, "Failed to get pod details", "pod", podName) + return false, nil } - result := false - - for i := 0; i < len(restResponse.Spec.Containers); i++ { - if strings.Contains(restResponse.Spec.Containers[0].Resources.Limits.CPU, expectedCPULimits) { - result = true - testenv.Log.Info("Verifying CPU limits: ", "pod", podName, "found", restResponse.Spec.Containers[0].Resources.Limits.CPU, "expected", expectedCPULimits) + for i := 0; i < len(podDetails.Spec.Containers); i++ { + if strings.Contains(podDetails.Spec.Containers[i].Resources.Limits.CPU, expectedCPULimits) { + testenv.Log.Info("Verifying CPU limits", "pod", podName, "found", podDetails.Spec.Containers[i].Resources.Limits.CPU, "expected", expectedCPULimits) + return true, nil } } - return result - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return false, nil + }) } -// VerifyResourceConstraints verifies value of CPU limits is as expected -func (testenv *TestCaseEnv) VerifyResourceConstraints(deployment *Deployment, podName string, res corev1.ResourceRequirements) { - gomega.Eventually(func() bool { - ns := testenv.GetName() - output, err := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() +// VerifyResourceConstraints verifies that all resource constraints (CPU/memory limits and requests) match on at least one container. +func (testenv *TestCaseEnv) VerifyResourceConstraints(deployment *Deployment, podName string, res corev1.ResourceRequirements) error { + return wait.PollUntilContextTimeout(context.TODO(), PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + podDetails, err := getPodDetails(testenv.GetName(), podName) if err != nil { - cmd := fmt.Sprintf("kubectl get pods -n %s %s -o json", ns, podName) - testenv.Log.Error(err, "Failed to execute command", "command", cmd) - return false - } - restResponse := PodDetailsStruct{} - err = json.Unmarshal([]byte(output), &restResponse) - if err != nil { - testenv.Log.Error(err, "Failed to parse JSON") - return false + testenv.Log.Error(err, "Failed to get pod details", "pod", podName) + return false, nil } - result := false - - for i := 0; i < len(restResponse.Spec.Containers); i++ { - if strings.Contains(restResponse.Spec.Containers[i].Resources.Limits.CPU, res.Limits.Cpu().String()) { - result = true - testenv.Log.Info("Verifying CPU limits: ", "pod", podName, "found", restResponse.Spec.Containers[0].Resources.Limits.CPU, "expected", res.Limits.Cpu().String()) - } - if strings.Contains(restResponse.Spec.Containers[i].Resources.Limits.Memory, res.Limits.Memory().String()) { - result = true - testenv.Log.Info("Verifying Memory limits: ", "pod", podName, "found", restResponse.Spec.Containers[i].Resources.Limits.Memory, "expected", res.Limits.Memory().String()) - } - - if strings.Contains(restResponse.Spec.Containers[i].Resources.Requests.CPU, res.Requests.Cpu().String()) { - result = true - testenv.Log.Info("Verifying CPU limits: ", "pod", podName, "found", restResponse.Spec.Containers[i].Resources.Requests.CPU, "expected", res.Requests.Cpu().String()) - } + for i := 0; i < len(podDetails.Spec.Containers); i++ { + c := podDetails.Spec.Containers[i] + cpuLimits := strings.Contains(c.Resources.Limits.CPU, res.Limits.Cpu().String()) + memLimits := strings.Contains(c.Resources.Limits.Memory, res.Limits.Memory().String()) + cpuRequests := strings.Contains(c.Resources.Requests.CPU, res.Requests.Cpu().String()) + memRequests := strings.Contains(c.Resources.Requests.Memory, res.Requests.Memory().String()) - if strings.Contains(restResponse.Spec.Containers[i].Resources.Requests.Memory, res.Requests.Memory().String()) { - result = true - testenv.Log.Info("Verifying CPU limits: ", "pod", podName, "found", restResponse.Spec.Containers[i].Resources.Requests.Memory, "expected", res.Requests.Memory().String()) + if cpuLimits && memLimits && cpuRequests && memRequests { + testenv.Log.Info("All resource constraints match", "pod", podName, + "cpuLimits", c.Resources.Limits.CPU, "memLimits", c.Resources.Limits.Memory, + "cpuRequests", c.Resources.Requests.CPU, "memRequests", c.Resources.Requests.Memory) + return true, nil } } - return result - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return false, nil + }) } -// VerifyClusterManagerPhase verify phase of cluster manager -func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { +// VerifyClusterManagerPhase verify phase of Cluster Manager +func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) error { cm := &enterpriseApi.ClusterManager{} - gomega.Eventually(func() enterpriseApi.Phase { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, deployment.GetName(), cm) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) + testenv.Log.Info("Waiting for Cluster Manager Phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase, "expected", phase) DumpGetPods(testenv.GetName()) - - // Test ClusterManager Phase to see if its ready - return cm.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).Should(gomega.Equal(phase)) + return cm.Status.Phase == phase, nil + }) } -// VerifyClusterMasterPhase verify phase of cluster manager -func (testenv *TestCaseEnv) VerifyClusterMasterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { +// VerifyClusterMasterPhase verify phase of Cluster Master +func (testenv *TestCaseEnv) VerifyClusterMasterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) error { cm := &enterpriseApiV3.ClusterMaster{} - gomega.Eventually(func() enterpriseApi.Phase { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, deployment.GetName(), cm) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) + testenv.Log.Info("Waiting for Cluster Master Phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase, "expected", phase) DumpGetPods(testenv.GetName()) - - // Test ClusterManager Phase to see if its ready - return cm.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).WithContext(ctx).Should(gomega.Equal(phase)) + return cm.Status.Phase == phase, nil + }) } // VerifySecretsOnPods Check whether the secret object info is mounted on given pods // Set match to true or false to indicate desired +ve or -ve match -func (testenv *TestCaseEnv) VerifySecretsOnPods(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySecretsOnPods(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) error { for _, pod := range verificationPods { for secretKey, secretValue := range data { found := false @@ -733,84 +732,102 @@ func (testenv *TestCaseEnv) VerifySecretsOnPods(ctx context.Context, deployment comparsion := bytes.Compare([]byte(currentValue), secretValue) if comparsion == 0 { found = true - testenv.Log.Info("Secret Values on POD Match", "Match Expected", match, "Pod Name ", pod, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", currentValue) + testenv.Log.Info("Secret Values on POD Match", "matchExpected", match, "podName", pod, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", currentValue) } else { - testenv.Log.Info("Secret Values on POD DONOT Match", "Match Expected", match, "Pod Name ", pod, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", currentValue) + testenv.Log.Info("Secret Values on POD DONOT Match", "matchExpected", match, "podName", pod, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", currentValue) + } + if found != match { + return fmt.Errorf("secret %s on pod %s: found=%v, expected=%v", secretKey, pod, found, match) } - gomega.Expect(found).Should(gomega.Equal(match)) } } + return nil } // VerifySecretsOnSecretObjects Compare secret value on passed in map to value present on secret object. // Set match to true or false to indicate desired +ve or -ve match -func (testenv *TestCaseEnv) VerifySecretsOnSecretObjects(ctx context.Context, deployment *Deployment, secretObjectNames []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySecretsOnSecretObjects(ctx context.Context, deployment *Deployment, secretObjectNames []string, data map[string][]byte, match bool) error { for _, secretName := range secretObjectNames { currentSecretData, err := GetSecretStruct(ctx, deployment, testenv.GetName(), secretName) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get secret struct") + if err != nil { + return fmt.Errorf("unable to get secret struct %s: %w", secretName, err) + } for secretKey, secretValue := range data { found := false secretValueOnSecretObject := currentSecretData.Data[secretKey] comparsion := bytes.Compare(secretValueOnSecretObject, secretValue) if comparsion == 0 { - testenv.Log.Info("Secret Values on Secret Object Match", "Match Expected", match, "Secret Object Name", secretName, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", string(secretValueOnSecretObject)) + testenv.Log.Info("Secret Values on Secret Object Match", "matchExpected", match, "secretObjectName", secretName, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", string(secretValueOnSecretObject)) found = true } else { - testenv.Log.Info("Secret Values on Secret Object DONOT match", "Match Expected", match, "Secret Object Name", secretName, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", string(secretValueOnSecretObject)) + testenv.Log.Info("Secret Values on Secret Object DONOT match", "matchExpected", match, "secretObjectName", secretName, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", string(secretValueOnSecretObject)) + } + if found != match { + return fmt.Errorf("secret %s on object %s: found=%v, expected=%v", secretKey, secretName, found, match) } - gomega.Expect(found).Should(gomega.Equal(match)) } } + return nil } // VerifySplunkServerConfSecrets Compare secret value on passed in map to value present on server.conf for given pods and secrets // Set match to true or false to indicate desired +ve or -ve match -func (testenv *TestCaseEnv) VerifySplunkServerConfSecrets(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySplunkServerConfSecrets(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) error { for _, podName := range verificationPods { keysToMatch := GetKeysToMatch(podName) - testenv.Log.Info("Verificaton Keys Set", "Pod Name", podName, "Keys To Compare", keysToMatch) + testenv.Log.Info("Verificaton Keys Set", "podName", podName, "keysToCompare", keysToMatch) for _, secretName := range keysToMatch { found := false stanza := SecretKeytoServerConfStanza[secretName] _, value, err := GetSecretFromServerConf(ctx, deployment, podName, testenv.GetName(), "pass4SymmKey", stanza) - gomega.Expect(err).To(gomega.Succeed(), "Secret not found in conf file", "Secret Name", secretName) + if err != nil { + return fmt.Errorf("secret %s not found in conf file on pod %s: %w", secretName, podName, err) + } comparsion := strings.Compare(value, string(data[secretName])) if comparsion == 0 { - testenv.Log.Info("Secret Values on server.conf Match", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on server.conf Match", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) found = true } else { - testenv.Log.Info("Secret Values on server.conf DONOT MATCH", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on server.conf DONOT MATCH", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) + } + if found != match { + return fmt.Errorf("secret %s on server.conf pod %s: found=%v, expected=%v", secretName, podName, found, match) } - gomega.Expect(found).Should(gomega.Equal(match)) } } + return nil } -// VerifySplunkInputConfSecrets Compare secret value on passed in map to value present on input.conf for given indexer or standalone pods +// VerifySplunkInputConfSecrets compares secret values on passed-in map to values present in input.conf for given Indexer or Standalone pods // Set match to true or false to indicate desired +ve or -ve match -func (testenv *TestCaseEnv) VerifySplunkInputConfSecrets(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySplunkInputConfSecrets(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) error { secretName := "hec_token" for _, podName := range verificationPods { if strings.Contains(podName, "standalone") || strings.Contains(podName, "indexer") { found := false - testenv.Log.Info("Key Verificaton", "Pod Name", podName, "Key", secretName) + testenv.Log.Info("Key Verificaton", "podName", podName, "key", secretName) stanza := SecretKeytoServerConfStanza[secretName] _, value, err := GetSecretFromInputsConf(ctx, deployment, podName, testenv.GetName(), "token", stanza) - gomega.Expect(err).To(gomega.Succeed(), "Secret not found in conf file", "Secret Name", secretName) + if err != nil { + return fmt.Errorf("secret %s not found in input.conf on pod %s: %w", secretName, podName, err) + } comparsion := strings.Compare(value, string(data[secretName])) if comparsion == 0 { - testenv.Log.Info("Secret Values on input.conf Match", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on input.conf Match", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) found = true } else { - testenv.Log.Info("Secret Values on input.conf DONOT MATCH", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on input.conf DONOT MATCH", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) + } + if found != match { + return fmt.Errorf("secret %s on input.conf pod %s: found=%v, expected=%v", secretName, podName, found, match) } - gomega.Expect(found).Should(gomega.Equal(match)) } } + return nil } // VerifySplunkSecretViaAPI check if keys can be used to access api i.e validate they are authentic -func (testenv *TestCaseEnv) VerifySplunkSecretViaAPI(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySplunkSecretViaAPI(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) error { var keysToMatch []string for _, podName := range verificationPods { if strings.Contains(podName, "standalone") || strings.Contains(podName, "indexer") { @@ -820,16 +837,19 @@ func (testenv *TestCaseEnv) VerifySplunkSecretViaAPI(ctx context.Context, deploy } for _, secretName := range keysToMatch { validKey := false - testenv.Log.Info("Key Verificaton", "Pod Name", podName, "Key", secretName) + testenv.Log.Info("Key Verificaton", "podName", podName, "key", secretName) validKey = CheckSecretViaAPI(ctx, deployment, podName, secretName, string(data[secretName])) - gomega.Expect(validKey).Should(gomega.Equal(match)) + if validKey != match { + return fmt.Errorf("secret %s via API on pod %s: valid=%v, expected=%v", secretName, podName, validKey, match) + } } } + return nil } // VerifyPVC verifies if PVC exists or not -func (testenv *TestCaseEnv) VerifyPVC(deployment *Deployment, ns string, pvcName string, expectedToExist bool, verificationTimeout time.Duration) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyPVC(pvcName string, expectedToExist bool, verificationTimeout time.Duration) error { + return wait.PollUntilContextTimeout(context.TODO(), PollInterval, verificationTimeout, true, func(ctx context.Context) (bool, error) { pvcExists := false pvcsList := DumpGetPvcs(testenv.GetName()) @@ -839,24 +859,27 @@ func (testenv *TestCaseEnv) VerifyPVC(deployment *Deployment, ns string, pvcName break } } - testenv.Log.Info("PVC Status Verified", "PVC", pvcName, "STATUS", pvcExists, "EXPECTED", expectedToExist) - return pvcExists - }, verificationTimeout, PollInterval).Should(gomega.Equal(expectedToExist)) + testenv.Log.Info("PVC Status Verified", "pvc", pvcName, "status", pvcExists, "expected", expectedToExist) + return pvcExists == expectedToExist, nil + }) } // VerifyPVCsPerDeployment verifies for a given deployment if PVCs (etc and var) exists -func (testenv *TestCaseEnv) VerifyPVCsPerDeployment(deployment *Deployment, deploymentType string, instances int, expectedtoExist bool, verificationTimeout time.Duration) { +func (testenv *TestCaseEnv) VerifyPVCsPerDeployment(deployment *Deployment, deploymentType string, instances int, expectedtoExist bool, verificationTimeout time.Duration) error { pvcKind := []string{"etc", "var"} for i := 0; i < instances; i++ { for _, pvcVolumeKind := range pvcKind { PvcName := fmt.Sprintf(PVCString, pvcVolumeKind, deployment.GetName(), deploymentType, i) - testenv.VerifyPVC(deployment, testenv.GetName(), PvcName, expectedtoExist, verificationTimeout) + if err := testenv.VerifyPVC(PvcName, expectedtoExist, verificationTimeout); err != nil { + return err + } } } + return nil } // VerifyAppInstalled verify that app of specific version is installed. Method assumes that app is installed in all CR's in namespace -func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, versionCheck bool, statusCheck string, checkupdated bool, clusterWideInstall bool) { +func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, versionCheck bool, statusCheck string, checkupdated bool, clusterWideInstall bool) error { // Fail-fast test: check first pod and first app before checking all pods if len(pods) > 0 && len(apps) > 0 { testenv.Log.Info("Running fail-fast test on first pod before checking all pods", "pod", pods[0], "app", apps[0]) @@ -865,7 +888,7 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * status, versionInstalled, err := GetPodAppStatus(ctx, deployment, firstPod, ns, firstApp, clusterWideInstall) if err != nil { - gomega.Expect(err).To(gomega.Succeed(), fmt.Sprintf("Test failed - app %s not accessible on pod %s. This indicates a fundamental issue. Skipping remaining checks.", firstApp, firstPod)) + return fmt.Errorf("test failed - app %s not accessible on pod %s: %w", firstApp, firstPod, err) } testenv.Log.Info("Test passed - app is accessible", "pod", firstPod, "app", firstApp, "status", status, "version", versionInstalled) testenv.Log.Info("Proceeding with full verification of all pods and apps") @@ -875,7 +898,9 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * for _, appName := range apps { status, versionInstalled, err := GetPodAppStatus(ctx, deployment, podName, ns, appName, clusterWideInstall) testenv.Log.Info("App details", "app", appName, "status", status, "version", versionInstalled, "error", err) - gomega.Expect(err).To(gomega.Succeed(), fmt.Sprintf("Unable to get app status on pod %s for app %s", podName, appName)) + if err != nil { + return fmt.Errorf("unable to get app status on pod %s: %w", podName, err) + } comparison := strings.EqualFold(status, statusCheck) //Check the app is installed on specific pods and un-installed on others for cluster-wide install var check bool @@ -883,7 +908,9 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * if strings.Contains(podName, "-indexer-") || strings.Contains(podName, "-search-head-") { check = true testenv.Log.Info("App Install Check", "pod", podName, "app", appName, "expected", check, "found", comparison, "scope:cluster", clusterWideInstall) - gomega.Expect(comparison).Should(gomega.Equal(check)) + if comparison != check { + return fmt.Errorf("app %s install check failed on pod %s: expected=%v, found=%v", appName, podName, check, comparison) + } } } else { // For local install check pods individually @@ -893,7 +920,9 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * check = true } testenv.Log.Info("App Install Check", "pod", podName, "app", appName, "expected", check, "found", comparison, "scope:cluster", clusterWideInstall) - gomega.Expect(comparison).Should(gomega.Equal(check)) + if comparison != check { + return fmt.Errorf("app %s install check failed on pod %s: expected=%v, found=%v", appName, podName, check, comparison) + } } if versionCheck { @@ -906,15 +935,18 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * expectedVersion = AppInfo[appName]["V1"] } testenv.Log.Info("Verify app", "pod", podName, "app", appName, "expectedVersion", expectedVersion, "versionInstalled", versionInstalled, "updated", checkupdated) - gomega.Expect(versionInstalled).Should(gomega.Equal(expectedVersion)) + if versionInstalled != expectedVersion { + return fmt.Errorf("app %s version mismatch on pod %s: expected=%s, found=%s", appName, podName, expectedVersion, versionInstalled) + } } } } } + return nil } // VerifyAppsCopied verify that apps are copied to correct location based on POD. Set checkAppDirectory false to verify app is not copied. -func (testenv *TestCaseEnv) VerifyAppsCopied(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, checkAppDirectory bool, scope string) { +func (testenv *TestCaseEnv) VerifyAppsCopied(ctx context.Context, deployment *Deployment, pods []string, apps []string, checkAppDirectory bool, scope string) error { for _, podName := range pods { path := "etc/apps" @@ -928,157 +960,201 @@ func (testenv *TestCaseEnv) VerifyAppsCopied(ctx context.Context, deployment *De path = splcommon.PeerAppsLoc } } - testenv.VerifyAppsInFolder(ctx, deployment, ns, podName, apps, path, checkAppDirectory) + if err := testenv.VerifyAppsInFolder(ctx, deployment, podName, apps, path, checkAppDirectory); err != nil { + return err + } } + return nil } // VerifyAppsInFolder verify that apps are present in folder. Set checkAppDirectory false to verify app is not copied. -func (testenv *TestCaseEnv) VerifyAppsInFolder(ctx context.Context, deployment *Deployment, ns string, podName string, apps []string, path string, checkAppDirectory bool) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyAppsInFolder(ctx context.Context, deployment *Deployment, podName string, apps []string, path string, checkAppDirectory bool) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { // Using checkAppDirectory here to get all files in case of negative check. GetDirsOrFilesInPath will return files/directory when checkAppDirecotry is FALSE appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, checkAppDirectory) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "Pod", podName) + if err != nil { + return false, fmt.Errorf("unable to get apps on pod %s: %w", podName, err) + } for _, app := range apps { folderName := app + "/" found := CheckStringInSlice(appList, folderName) testenv.Log.Info("App check", "pod", podName, "folderName", folderName, "path", path, "status", found) if found != checkAppDirectory { - return false + return false, nil } } - return true - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(true)) + return true, nil + }) } // VerifyAppsDownloadedOnContainer verify that apps are downloaded by init container -func (testenv *TestCaseEnv) VerifyAppsDownloadedOnContainer(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, path string) { +func (testenv *TestCaseEnv) VerifyAppsDownloadedOnContainer(ctx context.Context, deployment *Deployment, pods []string, apps []string, path string) error { for _, podName := range pods { appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, false) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "Pod", podName) + if err != nil { + return fmt.Errorf("unable to get apps on pod %s: %w", podName, err) + } for _, app := range apps { found := CheckStringInSlice(appList, app) - testenv.Log.Info("Check App files present on the pod", "Pod Name", podName, "App Name", app, "directory", path, "Status", found) - gomega.Expect(found).Should(gomega.Equal(true)) + testenv.Log.Info("Check App files present on the pod", "podName", podName, "appName", app, "directory", path, "status", found) + if !found { + return fmt.Errorf("app %s not found on pod %s in path %s", app, podName, path) + } } } + return nil } // VerifyAppsPackageDeletedOnOperatorContainer verify that apps are deleted by container -func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnOperatorContainer(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, path string) { +func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnOperatorContainer(ctx context.Context, deployment *Deployment, pods []string, apps []string, path string) error { for _, podName := range pods { for _, app := range apps { - gomega.Eventually(func() bool { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { appList, err := GetOperatorDirsOrFilesInPath(ctx, deployment, podName, path, false) if err != nil { - testenv.Log.Error(err, "Unable to get apps on operator pod", "Pod", podName) - return true + testenv.Log.Error(err, "Unable to get apps on operator pod", "pod", podName) + return false, nil } found := CheckStringInSlice(appList, app+"_") testenv.Log.Info(fmt.Sprintf("Check App package deleted on the pod %s. App Name %s. Directory %s, Status %t", podName, app, path, found)) - return found - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(false)) + return !found, nil + }) + if err != nil { + return fmt.Errorf("app package %s not deleted on operator pod %s: %w", app, podName, err) + } } } + return nil } // VerifyAppsPackageDeletedOnContainer verify that apps are deleted by container -func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnContainer(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, path string) { +func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnContainer(ctx context.Context, deployment *Deployment, pods []string, apps []string, path string) error { for _, podName := range pods { for _, app := range apps { - gomega.Eventually(func() bool { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, false) if err != nil { - testenv.Log.Error(err, "Unable to get apps on pod", "Pod", podName) - return true + testenv.Log.Error(err, "Unable to get apps on pod", "pod", podName) + return false, nil } found := CheckStringInSlice(appList, app+"_") testenv.Log.Info(fmt.Sprintf("Check App package deleted on the pod %s. App Name %s. Directory %s, Status %t", podName, app, path, found)) - return found - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(false)) + return !found, nil + }) + if err != nil { + return fmt.Errorf("app package %s not deleted on pod %s: %w", app, podName, err) + } } } + return nil } // VerifyAppListPhase verify given app Phase has completed for the given list of apps for given CR Kind -func (testenv *TestCaseEnv) VerifyAppListPhase(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, phase enterpriseApi.AppPhaseType, appList []string) { +func (testenv *TestCaseEnv) VerifyAppListPhase(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, phase enterpriseApi.AppPhaseType, appList []string) error { if phase == enterpriseApi.PhaseDownload || phase == enterpriseApi.PhasePodCopy { for _, appName := range appList { testenv.Log.Info(fmt.Sprintf("Check App Status for CR %s NAME %s APP NAME %s Expected Phase not to be %s", crKind, name, appName, phase)) - gomega.Eventually(func() enterpriseApi.AppPhaseType { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) if err != nil { testenv.Log.Error(err, "Failed to get app deployment info") - return phase // Continue polling + return false, nil // Continue polling } if appDeploymentInfo.AppName == "" { testenv.Log.Info(fmt.Sprintf("App deployment info not found yet for app %s (CR %s/%s, AppSource %s), continuing to poll", appName, crKind, name, appSourceName)) - return phase // Continue polling + return false, nil // Continue polling } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase should not be %s", crKind, name, appName, phase), "Actual Phase", appDeploymentInfo.PhaseInfo.Phase, "App State", appDeploymentInfo) - return appDeploymentInfo.PhaseInfo.Phase - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).ShouldNot(gomega.Equal(phase)) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase should not be %s", crKind, name, appName, phase), "actualPhase", appDeploymentInfo.PhaseInfo.Phase, "appState", appDeploymentInfo) + return appDeploymentInfo.PhaseInfo.Phase != phase, nil + }) + if err != nil { + return fmt.Errorf("app %s on CR %s/%s did not move past phase %s: %w", appName, crKind, name, phase, err) + } } } else { for _, appName := range appList { testenv.Log.Info(fmt.Sprintf("Check App Status for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase)) - gomega.Eventually(func() enterpriseApi.AppPhaseType { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) if err != nil { testenv.Log.Error(err, "Failed to get app deployment info") - return enterpriseApi.PhaseDownload // Continue polling + return false, nil // Continue polling } if appDeploymentInfo.AppName == "" { testenv.Log.Info(fmt.Sprintf("App deployment info not found yet for app %s (CR %s/%s, AppSource %s), continuing to poll", appName, crKind, name, appSourceName)) - return enterpriseApi.PhaseDownload // Continue polling + return false, nil // Continue polling } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase), "Actual Phase", appDeploymentInfo.PhaseInfo.Phase, "App Phase Status", appDeploymentInfo.PhaseInfo.Status, "App State", appDeploymentInfo) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase), "actualPhase", appDeploymentInfo.PhaseInfo.Phase, "appPhaseStatus", appDeploymentInfo.PhaseInfo.Status, "appState", appDeploymentInfo) if appDeploymentInfo.PhaseInfo.Status != enterpriseApi.AppPkgInstallComplete { - testenv.Log.Info("Phase Install Not Complete.", "Phase Found", appDeploymentInfo.PhaseInfo.Phase, "Phase Status Found", appDeploymentInfo.PhaseInfo.Status) - return enterpriseApi.PhaseDownload + testenv.Log.Info("Phase Install Not Complete.", "phaseFound", appDeploymentInfo.PhaseInfo.Phase, "phaseStatusFound", appDeploymentInfo.PhaseInfo.Status) + return false, nil } - return appDeploymentInfo.PhaseInfo.Phase - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(phase)) + return appDeploymentInfo.PhaseInfo.Phase == phase, nil + }) + if err != nil { + return fmt.Errorf("app %s on CR %s/%s did not reach phase %s: %w", appName, crKind, name, phase, err) + } } } + return nil } // VerifyAppState verify given app state is in between states passed as parameters, i.e when Status is between 101 and 303 we would pass enterpriseApi.AppPkgInstallComplete and enterpriseApi.AppPkgPodCopyComplete -func (testenv *TestCaseEnv) VerifyAppState(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string, appStateFinal enterpriseApi.AppPhaseStatusType, appStateInitial enterpriseApi.AppPhaseStatusType) { +func (testenv *TestCaseEnv) VerifyAppState(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string, appStateFinal enterpriseApi.AppPhaseStatusType, appStateInitial enterpriseApi.AppPhaseStatusType, timeout time.Duration) error { for _, appName := range appList { - gomega.Eventually(func() enterpriseApi.AppPhaseStatusType { - appDeploymentInfo, _ := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) - return appDeploymentInfo.PhaseInfo.Status - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.BeNumerically("~", appStateFinal, appStateInitial)) //Check status value is between appStateInitial and appStateFinal + err := wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, _ := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) + status := appDeploymentInfo.PhaseInfo.Status + // Check status value is approximately between appStateInitial and appStateFinal + diff := status - appStateFinal + if diff < 0 { + diff = -diff + } + return diff <= appStateInitial, nil + }) + if err != nil { + return fmt.Errorf("app %s state not in expected range: %w", appName, err) + } } + return nil } // WaitForAppInstall waits until an app is correctly installed (having status equal to 303) -func (testenv *TestCaseEnv) WaitForAppInstall(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string) { +func (testenv *TestCaseEnv) WaitForAppInstall(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string) error { for _, appName := range appList { - gomega.Eventually(func() enterpriseApi.AppPhaseStatusType { - appDeploymentInfo, _ := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) - return appDeploymentInfo.PhaseInfo.Status - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.BeEquivalentTo(enterpriseApi.AppPkgInstallComplete)) + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, _ := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) + return appDeploymentInfo.PhaseInfo.Status == enterpriseApi.AppPkgInstallComplete, nil + }) + if err != nil { + return fmt.Errorf("app %s did not reach install complete status: %w", appName, err) + } } - + return nil } // VerifyPodsInMCConfigMap checks if given pod names are present in given KEY of given MC's Config Map -func (testenv *TestCaseEnv) VerifyPodsInMCConfigMap(ctx context.Context, deployment *Deployment, pods []string, key string, mcName string, expected bool) { +func (testenv *TestCaseEnv) VerifyPodsInMCConfigMap(ctx context.Context, deployment *Deployment, pods []string, key string, mcName string, expected bool) error { // Get contents of MC config map mcConfigMap, err := GetMCConfigMap(ctx, deployment, testenv.GetName(), mcName) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get MC config map") + if err != nil { + return fmt.Errorf("unable to get MC config map: %w", err) + } for _, podName := range pods { - testenv.Log.Info("Checking for POD on MC Config Map", "POD Name", podName, "DATA", mcConfigMap.Data) - gomega.Expect(expected).To(gomega.Equal(CheckPodNameInString(podName, mcConfigMap.Data[key])), "Verify Pod in MC Config Map. Pod Name %s.", podName) + testenv.Log.Info("Checking for POD on MC Config Map", "podName", podName, "data", mcConfigMap.Data) + found := CheckPodNameInString(podName, mcConfigMap.Data[key]) + if found != expected { + return fmt.Errorf("verify pod in MC Config Map failed: pod %s, found=%v, expected=%v", podName, found, expected) + } } + return nil } // VerifyPodsInMCConfigString checks if given pod names are present in given KEY of given MC's Config Map -func (testenv *TestCaseEnv) VerifyPodsInMCConfigString(ctx context.Context, deployment *Deployment, pods []string, mcName string, expected bool, checkPodIP bool) { +func (testenv *TestCaseEnv) VerifyPodsInMCConfigString(ctx context.Context, pods []string, mcName string, expected bool, checkPodIP bool) error { for _, podName := range pods { - testenv.Log.Info("Checking pod configured in MC POD Peers String", "Pod Name", podName) + testenv.Log.Info("Checking pod configured in MC POD Peers String", "podName", podName) var found bool if checkPodIP { podIP := GetPodIP(testenv.GetName(), podName) @@ -1086,21 +1162,25 @@ func (testenv *TestCaseEnv) VerifyPodsInMCConfigString(ctx context.Context, depl } else { found = CheckPodNameOnMC(testenv.GetName(), mcName, podName) } - gomega.Expect(expected).To(gomega.Equal(found), "Verify Pod in MC Config String. Pod Name %s.", podName) + if found != expected { + return fmt.Errorf("verify pod in MC Config String failed: pod %s, found=%v, expected=%v", podName, found, expected) + } } + return nil } // VerifyClusterManagerBundlePush verify that bundle push was pushed on all indexers -func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, deployment *Deployment, ns string, replicas int, previousBundleHash string) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, deployment *Deployment, replicas int, previousBundleHash string) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { // Get Bundle status and check that each pod has successfully deployed the latest bundle - clusterManagerBundleStatus := CMBundlePushstatus(ctx, deployment, previousBundleHash, "cmanager") + cmEndpoint := "cmanager" if strings.Contains(deployment.GetName(), "master") { - clusterManagerBundleStatus = CMBundlePushstatus(ctx, deployment, previousBundleHash, "cmaster") + cmEndpoint = "cmaster" } + clusterManagerBundleStatus := CMBundlePushstatus(ctx, deployment, previousBundleHash, cmEndpoint) if len(clusterManagerBundleStatus) < replicas { - testenv.Log.Info("Bundle push on Pod not complete on all pods", "Pod with bundle push", clusterManagerBundleStatus) - return false + testenv.Log.Info("Bundle push on Pod not complete on all pods", "podWithBundlePush", clusterManagerBundleStatus) + return false, nil } clusterPodNames := DumpGetPods(testenv.GetName()) @@ -1108,43 +1188,43 @@ func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, if strings.Contains(podName, "-indexer-") { if _, present := clusterManagerBundleStatus[podName]; present { if clusterManagerBundleStatus[podName] != "Up" { - testenv.Log.Info("Bundle push on Pod not complete", "Pod Name", podName, "Status", clusterManagerBundleStatus[podName]) - return false + testenv.Log.Info("Bundle push on Pod not complete", "podName", podName, "status", clusterManagerBundleStatus[podName]) + return false, nil } } else { - testenv.Log.Info("Bundle push not found on pod", "Podname", podName) - return false + testenv.Log.Info("Bundle push not found on pod", "podName", podName) + return false, nil } } } - return true - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(true)) + return true, nil + }) } // VerifyDeployerBundlePush verify that bundle push was pushed on all search heads -func (testenv *TestCaseEnv) VerifyDeployerBundlePush(ctx context.Context, deployment *Deployment, ns string, replicas int) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyDeployerBundlePush(ctx context.Context, deployment *Deployment, ns string, replicas int) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { deployerAppPushStatus := DeployerBundlePushstatus(ctx, deployment, ns) if len(deployerAppPushStatus) == 0 { testenv.Log.Info("Bundle push not complete on all pods") DumpGetPods(testenv.GetName()) - return false + return false, nil } for appName, val := range deployerAppPushStatus { if val < replicas { - testenv.Log.Info("Bundle push not complete on all pods for", "App Name", appName, "Replicas with bundle push", val, "Expected replicas", replicas) + testenv.Log.Info("Bundle push not complete on all pods for", "appName", appName, "replicasWithBundlePush", val, "expectedReplicas", replicas) DumpGetPods(testenv.GetName()) - return false + return false, nil } } - return true - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(true)) + return true, nil + }) } // VerifyNoPodResetByUID verify that no pod reset during App install by comparing pod UIDs -func (testenv *TestCaseEnv) VerifyNoPodResetByUID(ctx context.Context, deployment *Deployment, podUIDMap map[string]string, podToSkip []string) { +func (testenv *TestCaseEnv) VerifyNoPodResetByUID(ctx context.Context, podUIDMap map[string]string, podToSkip []string) error { if podUIDMap == nil { testenv.Log.Info("podUIDMap is empty. Skipping validation") } else { @@ -1156,78 +1236,91 @@ func (testenv *TestCaseEnv) VerifyNoPodResetByUID(ctx context.Context, deploymen testenv.Log.Info("Checking Pod reset for Pod Name", "podName", podName, "currentUID", currentUID) if previousUID, ok := podUIDMap[podName]; ok { if !CheckStringInSlice(podToSkip, podName) { - gomega.Expect(currentUID).To(gomega.Equal(previousUID), "Pod reset was detected. Pod Name %s. Current Pod UID %s. Previous Pod UID %s", podName, currentUID, previousUID) + if currentUID != previousUID { + return fmt.Errorf("pod reset was detected. Pod Name %s. Current Pod UID %s. Previous Pod UID %s", podName, currentUID, previousUID) + } } } } } + return nil } -// WaitForSplunkPodCleanup Wait for cleanup to happend -func (testenv *TestCaseEnv) WaitForSplunkPodCleanup(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() int { +// WaitForSplunkPodCleanup Wait for cleanup to happen +func (testenv *TestCaseEnv) WaitForSplunkPodCleanup(ctx context.Context, deployment *Deployment) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { testenv.Log.Info("Waiting for Splunk Pods to be deleted before running test") - return len(DumpGetPods(testenv.GetName())) - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(0)) + return len(DumpGetPods(testenv.GetName())) == 0, nil + }) } // WaitforAppInstallState Wait for App to reach state specified in conf file -func (testenv *TestCaseEnv) WaitforAppInstallState(ctx context.Context, deployment *Deployment, podNames []string, ns string, appName string, newState string, clusterWideInstall bool) { +func (testenv *TestCaseEnv) WaitforAppInstallState(ctx context.Context, deployment *Deployment, podNames []string, ns string, appName string, newState string, clusterWideInstall bool) error { testenv.Log.Info("Retrieve App state on pod") for _, podName := range podNames { - gomega.Eventually(func() string { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { status, _, err := GetPodAppStatus(ctx, deployment, podName, ns, appName, clusterWideInstall) testenv.Log.Info("App details", "app", appName, "status", status, "error", err, "podName", podName) - return status - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(strings.ToUpper(newState))) + return status == strings.ToUpper(newState), nil + }) + if err != nil { + return fmt.Errorf("app %s did not reach state %s on pod %s: %w", appName, newState, podName, err) + } } + return nil } // VerifyAppRepoState verify given app repo state is equal to given value for app for given CR Kind -func (testenv *TestCaseEnv) VerifyAppRepoState(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, repoValue int, appName string) { +func (testenv *TestCaseEnv) VerifyAppRepoState(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, repoValue int, appName string) error { testenv.Log.Info("Check for app repo state in CR") - gomega.Eventually(func() int { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) if err != nil { testenv.Log.Error(err, "Failed to get app deployment info") - return 0 + return false, nil } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected repo value %d", crKind, name, appName, repoValue), "Actual Value", appDeploymentInfo.RepoState, "App State", appDeploymentInfo) - return int(appDeploymentInfo.RepoState) - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(repoValue)) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected repo value %d", crKind, name, appName, repoValue), "actualValue", appDeploymentInfo.RepoState, "appState", appDeploymentInfo) + return int(appDeploymentInfo.RepoState) == repoValue, nil + }) } // VerifyIsDeploymentInProgressFlagIsSet verify IsDeploymentInProgress flag is set to true -func (testenv *TestCaseEnv) VerifyIsDeploymentInProgressFlagIsSet(ctx context.Context, deployment *Deployment, name string, crKind string) { - testenv.Log.Info("Check IsDeploymentInProgress Flag is set", "CR NAME", name, "CR Kind", crKind) - gomega.Eventually(func() bool { - isDeploymentInProgress, err := GetIsDeploymentInProgressFlag(ctx, deployment, testenv, name, crKind) +func (testenv *TestCaseEnv) VerifyIsDeploymentInProgressFlagIsSet(ctx context.Context, deployment *Deployment, name string, crKind string) error { + testenv.Log.Info("Check IsDeploymentInProgress Flag is set", "crName", name, "crKind", crKind) + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + isDeploymentInProgress, err := testenv.GetIsDeploymentInProgressFlag(ctx, deployment, name, crKind) if err != nil { testenv.Log.Error(err, "Failed to get isDeploymentInProgress Flag") - return false + return false, nil } - testenv.Log.Info("IsDeploymentInProgress Flag status found", "CR NAME", name, "CR Kind", crKind, "IsDeploymentInProgress", isDeploymentInProgress) - return isDeploymentInProgress - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(true)) + testenv.Log.Info("IsDeploymentInProgress Flag status found", "crName", name, "crKind", crKind, "isDeploymentInProgress", isDeploymentInProgress) + return isDeploymentInProgress, nil + }) } // VerifyFilesInDirectoryOnPod verify that files are present in folder. -func (testenv *TestCaseEnv) VerifyFilesInDirectoryOnPod(ctx context.Context, deployment *Deployment, podNames []string, files []string, path string, checkDirectory bool, checkPresent bool) { +func (testenv *TestCaseEnv) VerifyFilesInDirectoryOnPod(ctx context.Context, deployment *Deployment, podNames []string, files []string, path string, checkDirectory bool, checkPresent bool) error { for _, podName := range podNames { - gomega.Eventually(func() bool { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { // Using checkDirectory here to get all files in case of negative check. GetDirsOrFilesInPath will return files/directory when checkDirecotry is FALSE filelist, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, checkDirectory) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get files on pod", "Pod", podName) + if err != nil { + return false, fmt.Errorf("unable to get files on pod %s: %w", podName, err) + } for _, file := range files { found := CheckStringInSlice(filelist, file) testenv.Log.Info("File check", "pod", podName, "filename", file, "path", path, "status", found) if found != checkPresent { - return false + return false, nil } } - return true - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(true)) + return true, nil + }) + if err != nil { + return err + } } + return nil } func (testenv *TestCaseEnv) GetTelemetryLastSubmissionTime(ctx context.Context, deployment *Deployment) string { @@ -1262,16 +1355,16 @@ func (testenv *TestCaseEnv) GetTelemetryLastSubmissionTime(ctx context.Context, } // VerifyTelemetry checks that the telemetry ConfigMap has a non-empty lastTransmission field in its status key. -func (testenv *TestCaseEnv) VerifyTelemetry(ctx context.Context, deployment *Deployment, prevVal string) { +func (testenv *TestCaseEnv) VerifyTelemetry(ctx context.Context, deployment *Deployment, prevVal string) error { testenv.Log.Info("VerifyTelemetry: start") - gomega.Eventually(func() bool { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { currentVal := testenv.GetTelemetryLastSubmissionTime(ctx, deployment) if currentVal != "" && currentVal != prevVal { testenv.Log.Info("VerifyTelemetry: success", "previous", prevVal, "current", currentVal) - return true + return true, nil } - return false - }, deployment.GetTimeout(), PollInterval).WithContext(ctx).Should(gomega.Equal(true)) + return false, nil + }) } // TriggerTelemetrySubmission updates or adds the 'test_submission' key in the telemetry ConfigMap with a JSON value containing a random number. @@ -1369,7 +1462,7 @@ func (testenv *TestCaseEnv) WaitForPodsInMCConfigMap(ctx context.Context, deploy } // WaitForPodsInMCConfigString waits for pods to appear in MC config string -func (testenv *TestCaseEnv) WaitForPodsInMCConfigString(ctx context.Context, deployment *Deployment, pods []string, mcName string, expected bool, checkPodIP bool, timeout time.Duration) error { +func (testenv *TestCaseEnv) WaitForPodsInMCConfigString(ctx context.Context, pods []string, mcName string, expected bool, checkPodIP bool, timeout time.Duration) error { return wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { for _, podName := range pods { var found bool @@ -1420,7 +1513,7 @@ func (testenv *TestCaseEnv) WaitForIndexerClusterPhase(ctx context.Context, depl // WaitForSearchResultsNonEmpty waits for search results to return a non-empty "result" field func WaitForSearchResultsNonEmpty(ctx context.Context, deployment *Deployment, podName string, searchString string, timeout time.Duration) error { return wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { - searchResultsResp, err := PerformSearchSync(ctx, podName, searchString, deployment) + searchResultsResp, err := PerformSearchSync(ctx, deployment, podName, searchString) if err != nil { return false, nil } @@ -1537,7 +1630,7 @@ func (testenv *TestCaseEnv) WaitForAppRepoStateChange(ctx context.Context, deplo } } - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, crName, crKind, appSourceName, lookupAppName) + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, crName, crKind, appSourceName, lookupAppName) if err != nil { testenv.Log.Info("Failed to get app deployment info while waiting for repo state change", "app", appName, "error", err) return false, nil @@ -1563,3 +1656,409 @@ func (testenv *TestCaseEnv) WaitForAppRepoStateChange(ctx context.Context, deplo return false, nil }) } + +// VerifyC3ClusterPVCs verifies that PVCs for SHC, Deployer, Indexers, and Cluster Manager exist or are deleted. +func VerifyC3ClusterPVCs(testcaseEnvInst *TestCaseEnv, deployment *Deployment, clusterManagerType string, exists bool, timeout time.Duration) error { + if err := testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, exists, timeout); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, exists, timeout); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, exists, timeout); err != nil { + return err + } + return testcaseEnvInst.VerifyPVCsPerDeployment(deployment, clusterManagerType, 1, exists, timeout) +} + +// VerifyM4ClusterAndRFSF verifies Cluster Manager and multisite cluster are ready and RF/SF is met. +// When skipMultisiteStatus is true the VerifyIndexerClusterMultisiteStatus check is omitted +// (useful on second-round verification after an index addition where multisite topology is unchanged). +func VerifyM4ClusterAndRFSF(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig, siteCount int, skipMultisiteStatus bool) error { + if err := config.ClusterManagerReady(ctx, deployment, testcaseEnvInst); err != nil { + return err + } + if err := testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + if !skipMultisiteStatus { + if err := testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount); err != nil { + return err + } + } + if err := testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment); err != nil { + return err + } + return testcaseEnvInst.VerifyRFSFMet(ctx, deployment) +} + +// VerifyLMAppsOnPod verifies that apps are copied and installed on the License Manager pod. +// The updated flag controls whether apps are expected to be updated versions. +func VerifyLMAppsOnPod(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, testenvInstance *TestEnv, podName []string, appList []string, updated bool) error { + if err := testcaseEnvInst.VerifyAppsCopied(ctx, deployment, podName, appList, true, enterpriseApi.ScopeLocal); err != nil { + return err + } + return testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appList, updated, "enabled", updated, false) +} + +// VerifyLMConfiguredOnCluster verifies that the License Manager is configured on +// the given indexer pods, on search head pods, and on the Monitoring Console. +func VerifyLMConfiguredOnCluster(ctx context.Context, deployment *Deployment, indexerPods []string) error { + shPods := GeneratePodNameSlice(SearchHeadPod, deployment.GetName(), 3, false, 0) + if err := VerifyLMConfiguredOnPods(ctx, deployment, append(indexerPods, shPods...)); err != nil { + return err + } + return VerifyLMConfiguredOnMC(ctx, deployment) +} + +// VerifyMCConfigForCluster verifies that the CM, deployer, search heads, and indexers +// are all correctly registered in the MC config map and pod config string. +// It uses the service name and URL key from the MCVersionConfig so it works for both V3 and V4. +func VerifyMCConfigForCluster(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + cfg MCVersionConfig, mcName string, shPods, indexerPods []string) error { + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(cfg.CMServiceNameFmt, deployment.GetName())}, cfg.CMURLKey, mcName, true); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcName, true, false); err != nil { + return err + } + return testcaseEnvInst.VerifyPodsInMCConfigString(ctx, indexerPods, mcName, true, true) +} + +// VerifyStandalonePodsInMC verifies that the given standalone pods are present (or absent) in the +// MC config map and pod config string. +func VerifyStandalonePodsInMC(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, pods []string, mcName string, shouldExist bool) error { + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, pods, "SPLUNK_STANDALONE_URL", mcName, shouldExist); err != nil { + return err + } + return testcaseEnvInst.VerifyPodsInMCConfigString(ctx, pods, mcName, shouldExist, false) +} + +// VerifyMCTwoAfterCMReconfig verifies that MC Two is correctly configured after the Cluster Manager +// has been reconfigured to point to it: CM and indexers should be present, SH should be absent. +// If checkDeployerAbsent is true, also verifies deployer is absent on MC Two (used in C3 tests). +func VerifyMCTwoAfterCMReconfig(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, checkDeployerAbsent bool) error { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, indexerPods, mcTwoName, true, true); err != nil { + return err + } + + if checkDeployerAbsent { + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC Two Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false); err != nil { + return err + } + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config String after CM Reconfig") + return testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcTwoName, false, false) +} + +// VerifyMCOneAfterCMReconfig verifies that MC One is correctly configured after the Cluster Manager +// has been reconfigured away from it: CM should be absent, SH should still be present. +// If checkDeployerPresent is true, also verifies deployer is still present on MC One (used in M4 tests). +func VerifyMCOneAfterCMReconfig(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, checkDeployerPresent bool) error { + + if err := testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false); err != nil { + return err + } + + // CSPL-619: Indexer verification on MC One is commented out in all test variants + + if checkDeployerPresent { + testcaseEnvInst.Log.Info("Verify Deployer still in MC One Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true); err != nil { + return err + } + } + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config String after CM Reconfig") + return testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcName, true, false) +} + +// VerifyMCTwoAfterSHCReconfig verifies that MC Two has all components (CM, deployer, SH, indexers) +// after the SHC has been reconfigured to point to it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCTwoAfterSHCReconfig(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, timeout time.Duration) error { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify Deployer in MC Two Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true); err != nil { + return err + } + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig (with wait)") + if err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, shPods, mcTwoName, true, false, timeout); err != nil { + return fmt.Errorf("timed out waiting for search heads in MC two config after SHC reconfig: %w", err) + } + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig (with wait)") + if err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, indexerPods, mcTwoName, true, true, timeout); err != nil { + return fmt.Errorf("timed out waiting for indexers in MC two config after SHC reconfig: %w", err) + } + } else { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcTwoName, true, false); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, indexerPods, mcTwoName, true, true); err != nil { + return err + } + } + return nil +} + +// VerifyMCOneAfterSHCReconfig verifies that MC One has lost all components (CM, deployer, SH) +// after the SHC has been reconfigured away from it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCOneAfterSHCReconfig(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, timeout time.Duration) error { + + if err := testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC One Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false); err != nil { + return err + } + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig (with wait)") + if err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, shPods, mcName, false, false, timeout); err != nil { + return fmt.Errorf("timed out waiting for search heads to be removed from MC one config after SHC reconfig: %w", err) + } + } else { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcName, false, false); err != nil { + return err + } + } + + // CSPL-619: Indexer verification on MC One is commented out in all test variants + return nil +} + +// VerifySecretsPropagated checks that the given secret data has been propagated to all +// versioned secret objects, pods, server config, input config, and via the API. +func VerifySecretsPropagated(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, secretData map[string][]byte, updated bool) error { + // Once Pods are READY check each versioned secret for updated secret keys + secretObjectNames := GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) + + // Verify Secrets on versioned secret objects + if err := testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretData, updated); err != nil { + return err + } + + // Once Pods are READY check each pod for updated secret keys + verificationPods := DumpGetPods(testcaseEnvInst.GetName()) + + // Verify secrets on pods + if err := testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretData, updated); err != nil { + return err + } + + // Verify Secrets on ServerConf on Pod + if err := testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretData, updated); err != nil { + return err + } + + // Verify Hec token on InputConf on Pod + if err := testcaseEnvInst.VerifySplunkInputConfSecrets(ctx, deployment, verificationPods, secretData, updated); err != nil { + return err + } + + // Verify Secrets via api access on Pod + return testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretData, updated) +} + +// S1WithLMSetup holds the resources created by SetupS1WithLMAndMC so that +// individual test functions can operate on them without repeating the setup. +type S1WithLMSetup struct { + Standalone *enterpriseApi.Standalone + Mc *enterpriseApi.MonitoringConsole + ResourceVersion string + NamespaceScopedSecretName string +} + +// SetupS1WithLMAndMC performs the common S1 setup shared by the secret-update +// and secret-delete tests: license config map, standalone with LM, MC, and +// initial secret verification. +func SetupS1WithLMAndMC(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig) (S1WithLMSetup, error) { + if err := SetupLicenseConfigMap(ctx, testcaseEnvInst); err != nil { + return S1WithLMSetup{}, err + } + + mcRef := deployment.GetName() + standalone, err := config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) + if err != nil { + return S1WithLMSetup{}, fmt.Errorf("unable to deploy standalone instance with LM: %w", err) + } + + if err := VerifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone); err != nil { + return S1WithLMSetup{}, fmt.Errorf("LM or standalone not ready: %w", err) + } + + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + if err != nil { + return S1WithLMSetup{}, fmt.Errorf("unable to deploy Monitoring Console: %w", err) + } + + namespaceScopedSecretName := fmt.Sprintf(NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + if _, err = GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName); err != nil { + return S1WithLMSetup{}, fmt.Errorf("unable to get secret struct: %w", err) + } + + return S1WithLMSetup{ + Standalone: standalone, + Mc: mc, + ResourceVersion: resourceVersion, + NamespaceScopedSecretName: namespaceScopedSecretName, + }, nil +} + +// VerifyLMAndStandaloneReady waits for License Manager then Standalone to reach READY status. +func VerifyLMAndStandaloneReady(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig, standalone *enterpriseApi.Standalone) error { + if err := config.LicenseManagerReady(ctx, deployment, testcaseEnvInst); err != nil { + return err + } + return testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) +} + +// VerifyLMAndClusterManagerReady waits for License Manager then Cluster Manager to reach READY status. +func VerifyLMAndClusterManagerReady(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig) error { + if err := config.LicenseManagerReady(ctx, deployment, testcaseEnvInst); err != nil { + return err + } + return config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) +} + +// VerifyS1SecretChangeApplied verifies that a secret change (update or delete) +// has been applied to the S1 stack: standalone enters Updating phase, LM and +// standalone return to Ready, MC version changes, and secrets are propagated. +func VerifyS1SecretChangeApplied(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig, setup S1WithLMSetup, secretData map[string][]byte, updated bool) error { + if err := testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseUpdating); err != nil { + return err + } + if err := VerifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, setup.Standalone); err != nil { + return err + } + if err := testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, setup.Mc, setup.ResourceVersion); err != nil { + return err + } + return VerifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretData, updated) +} + +// VerifyPostSecretChangeCluster performs the common tail verification after a +// secret change on a clustered deployment: MC version changed, RF/SF met, and +// secrets propagated to all pods. +func VerifyPostSecretChangeCluster(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, mc *enterpriseApi.MonitoringConsole, resourceVersion string, updatedSecretData map[string][]byte) error { + if err := testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Checking RF SF after secret change") + if err := testcaseEnvInst.VerifyRFSFMet(ctx, deployment); err != nil { + return err + } + + return VerifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} + +// VerifyConfFileContent retrieves a conf file from a pod and validates its content. +func VerifyConfFileContent(pod, confPath, deploymentName string, expectedContent []string, errorMsg string) error { + conf, err := GetConfFile(pod, confPath, deploymentName) + if err != nil { + return fmt.Errorf("%s: %w", errorMsg, err) + } + return ValidateContent(conf, expectedContent, true) +} + +// ApplySecretUpdateAndVerifyCMUpdating deploys MC, verifies RF/SF and initial secret state, +// applies a secret update, and confirms the Cluster Manager enters the Updating phase. +// Returns the MC, its resource version, and the updated secret data for post-change verification. +func ApplySecretUpdateAndVerifyCMUpdating(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig) (*enterpriseApi.MonitoringConsole, string, map[string][]byte, error) { + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + if err != nil { + return nil, "", nil, fmt.Errorf("unable to deploy Monitoring Console: %w", err) + } + testcaseEnvInst.Log.Info("Checking RF SF before secret change") + if err := testcaseEnvInst.VerifyRFSFMet(ctx, deployment); err != nil { + return nil, "", nil, err + } + namespaceScopedSecretName := fmt.Sprintf(NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + _, err = GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + if err != nil { + return nil, "", nil, fmt.Errorf("unable to get secret struct: %w", err) + } + updatedSecretData, err := GenerateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + if err != nil { + return nil, "", nil, fmt.Errorf("unable to generate and apply secret update: %w", err) + } + if err := config.VerifyClusterManagerPhaseUpdating(ctx, deployment, testcaseEnvInst); err != nil { + return nil, "", nil, err + } + return mc, resourceVersion, updatedSecretData, nil +} diff --git a/test/testenv/watch_utils.go b/test/testenv/watch_utils.go index e9b455164..45d3b963e 100644 --- a/test/testenv/watch_utils.go +++ b/test/testenv/watch_utils.go @@ -119,7 +119,7 @@ func (testenv *TestCaseEnv) WatchForIngestorClusterPhase(ctx context.Context, de // WatchForAppPhaseChange uses optimized polling to wait for app phase changes on a CR func (testenv *TestCaseEnv) WatchForAppPhaseChange(ctx context.Context, deployment *Deployment, namespace, crName, crKind, appSourceName, appName string, expectedPhase enterpriseApi.AppPhaseType, timeout time.Duration) error { return wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, crName, crKind, appSourceName, appName) + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, crName, crKind, appSourceName, appName) if err != nil { testenv.Log.Info("Failed to get app deployment info", "app", appName, "error", err) return false, nil @@ -144,7 +144,7 @@ func (testenv *TestCaseEnv) WatchForAllAppsPhaseChange(ctx context.Context, depl } } - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, crName, crKind, appSourceName, lookupAppName) + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, crName, crKind, appSourceName, lookupAppName) if err != nil { testenv.Log.Info("Failed to get app deployment info", "app", appName, "error", err) return false, nil