diff --git a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml index ee756236f..226830c31 100644 --- a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml +++ b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml @@ -136,7 +136,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-AL2023-int-test-workflow.yml b/.github/workflows/arm-AL2023-int-test-workflow.yml index 39986ebca..a65cf941f 100644 --- a/.github/workflows/arm-AL2023-int-test-workflow.yml +++ b/.github/workflows/arm-AL2023-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-al2023 diff --git a/.github/workflows/arm-RHEL-build-test-push-workflow.yml b/.github/workflows/arm-RHEL-build-test-push-workflow.yml index 07848683c..f86a5e632 100644 --- a/.github/workflows/arm-RHEL-build-test-push-workflow.yml +++ b/.github/workflows/arm-RHEL-build-test-push-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-RHEL-int-test-workflow.yml b/.github/workflows/arm-RHEL-int-test-workflow.yml index 83c7e8fb9..d2a127ca8 100644 --- a/.github/workflows/arm-RHEL-int-test-workflow.yml +++ b/.github/workflows/arm-RHEL-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml index a22bd10c4..5c6179c20 100644 --- a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml +++ b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml @@ -136,7 +136,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-Ubuntu-int-test-workflow.yml b/.github/workflows/arm-Ubuntu-int-test-workflow.yml index b3f133abe..8d138f5bb 100644 --- a/.github/workflows/arm-Ubuntu-int-test-workflow.yml +++ b/.github/workflows/arm-Ubuntu-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-ubuntu diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index 01e655adb..dda66569a 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -200,7 +200,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-build-test-push-workflow.yml b/.github/workflows/distroless-build-test-push-workflow.yml index 8aa6172f7..63d9bee33 100644 --- a/.github/workflows/distroless-build-test-push-workflow.yml +++ b/.github/workflows/distroless-build-test-push-workflow.yml @@ -195,7 +195,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-int-test-workflow.yml b/.github/workflows/distroless-int-test-workflow.yml index 01a024247..5f16a8472 100644 --- a/.github/workflows/distroless-int-test-workflow.yml +++ b/.github/workflows/distroless-int-test-workflow.yml @@ -73,7 +73,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-distroless diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 001a34cee..c545b69eb 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -70,7 +70,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/.github/workflows/manual-int-test-workflow.yml b/.github/workflows/manual-int-test-workflow.yml index 96629316e..0e3e5d357 100644 --- a/.github/workflows/manual-int-test-workflow.yml +++ b/.github/workflows/manual-int-test-workflow.yml @@ -29,7 +29,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/namespace-scope-int-workflow.yml b/.github/workflows/namespace-scope-int-workflow.yml index fc3d3554f..640f74dc6 100644 --- a/.github/workflows/namespace-scope-int-workflow.yml +++ b/.github/workflows/namespace-scope-int-workflow.yml @@ -25,7 +25,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml index 3f886dd9a..184927f04 100644 --- a/.github/workflows/nightly-int-test-workflow.yml +++ b/.github/workflows/nightly-int-test-workflow.yml @@ -66,7 +66,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/test/appframework_aws/c3/appframework_aws_suite_test.go b/test/appframework_aws/c3/appframework_aws_suite_test.go index 6468b018a..2fff7b4d3 100644 --- a/test/appframework_aws/c3/appframework_aws_suite_test.go +++ b/test/appframework_aws/c3/appframework_aws_suite_test.go @@ -17,24 +17,13 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/joho/godotenv" "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -42,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -55,87 +41,17 @@ var ( func TestBasic(t *testing.T) { RegisterFailHandler(Fail) - // Find and load the .env file from the current directory upwards - err := loadEnvFile() - Expect(err).ToNot(HaveOccurred(), "Error loading .env file") - - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) -} - -//func TestMain(m *testing.M) { -// Run the tests -// os.Exit(m.Run()) -//} - -// loadEnvFile traverses up the directory tree to find a .env file -func loadEnvFile() error { - // Get the current working directory - dir, err := os.Getwd() - if err != nil { - return err - } + Expect(testenv.LoadEnvFile()).ToNot(HaveOccurred(), "Error loading .env file") - // Traverse up the directory tree - for { - // Check if .env file exists in the current directory - envFile := filepath.Join(dir, ".env") - if _, err := os.Stat(envFile); err == nil { - // .env file found, load it - return godotenv.Load(envFile) - } - - // Move up to the parent directory - parentDir := filepath.Dir(dir) - if parentDir == dir { - // Reached the root directory - return nil - } - dir = parentDir - } + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework_aws/c3/appframework_aws_test.go index 857ba1742..4cda6ff5d 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework_aws/c3/appframework_aws_test.go @@ -25,7 +25,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -53,44 +52,13 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 4000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master", testenv.WithTimeout(4000)) + Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -138,7 +106,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -174,13 +142,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -193,23 +161,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -237,10 +195,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on S3 @@ -272,24 +231,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -305,10 +255,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -355,10 +306,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -395,13 +346,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -414,23 +365,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -446,10 +387,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############## DOWNGRADE APPS ############### // Delete apps on S3 @@ -481,24 +423,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -514,10 +447,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -577,13 +511,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -593,14 +527,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -632,7 +560,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -642,7 +571,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -650,9 +579,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -670,8 +597,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -693,12 +619,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -709,7 +630,7 @@ var _ = Describe("c3appfw test", func() { // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -727,7 +648,8 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -740,14 +662,12 @@ var _ = Describe("c3appfw test", func() { // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -763,8 +683,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -789,7 +708,7 @@ var _ = Describe("c3appfw test", func() { // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -806,15 +725,16 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -858,13 +778,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -874,14 +794,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -898,10 +812,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -921,16 +836,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -946,14 +855,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1021,7 +931,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1031,9 +941,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1044,8 +954,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1063,14 +973,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1089,10 +993,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on S3 @@ -1125,16 +1030,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1156,15 +1055,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1230,7 +1130,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1240,8 +1140,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1251,8 +1151,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1270,14 +1170,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1305,7 +1199,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1315,7 +1210,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete apps on S3 @@ -1348,16 +1243,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1379,15 +1268,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1413,7 +1303,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1432,13 +1322,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1448,14 +1338,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1472,10 +1356,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1526,7 +1411,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1560,29 +1445,23 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1604,10 +1483,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -1635,16 +1515,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1717,14 +1591,15 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1772,13 +1647,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1787,14 +1662,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1811,10 +1680,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -1834,22 +1704,17 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -1900,14 +1765,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1973,7 +1839,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1983,8 +1849,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1994,8 +1860,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2012,14 +1878,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2038,10 +1898,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on S3 @@ -2086,16 +1947,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2124,10 +1979,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2162,7 +2018,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2187,7 +2043,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2206,13 +2062,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2291,7 +2147,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2316,7 +2172,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2335,13 +2191,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2372,11 +2228,7 @@ var _ = Describe("c3appfw test", func() { // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) @@ -2419,7 +2271,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2437,13 +2289,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2456,16 +2308,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2482,10 +2328,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2513,7 +2360,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2531,13 +2378,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2550,16 +2397,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2576,10 +2417,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2622,13 +2464,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2637,14 +2479,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2660,7 +2496,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] @@ -2672,7 +2509,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) @@ -2726,7 +2563,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2743,13 +2580,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2785,16 +2622,10 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2809,12 +2640,13 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2834,7 +2666,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2843,7 +2675,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2854,7 +2686,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2863,14 +2695,14 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -2881,14 +2713,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2905,7 +2731,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2932,7 +2759,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2950,13 +2777,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2969,19 +2796,13 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2998,10 +2819,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -3036,13 +2858,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3061,18 +2883,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3129,11 +2947,11 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -3156,14 +2974,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3182,10 +3000,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -3193,10 +3011,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) // Deploy C3 SVA // Deploy the Cluster Master @@ -3239,14 +3057,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3259,19 +3071,21 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // //############### UPGRADE APPS ################ // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster @@ -3281,7 +3095,7 @@ var _ = Describe("c3appfw test", func() { // uploadedApps = append(uploadedApps, uploadedFiles...) // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) + // testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) // // Ensure that the Cluster Master goes to Ready phase // testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework_aws/c3/manager_appframework_test.go index a49fd3eb4..6fdf74430 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework_aws/c3/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,44 +50,13 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -136,7 +104,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -172,13 +140,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -191,23 +159,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -235,10 +193,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on S3 @@ -270,24 +229,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -303,10 +253,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -331,7 +282,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) default: fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with provider set as %v", testenv.ClusterProvider)) } // Upload V1 apps to S3 for Monitoring Console @@ -393,7 +344,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE IMAGE ################ @@ -496,10 +447,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -536,13 +487,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -555,23 +506,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -587,10 +528,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############## DOWNGRADE APPS ############### // Delete apps on S3 @@ -622,24 +564,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -655,10 +588,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -718,13 +652,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -734,14 +668,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -773,7 +701,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -783,7 +712,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -791,9 +720,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -811,8 +738,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -838,12 +764,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -854,7 +775,7 @@ var _ = Describe("c3appfw test", func() { // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -872,7 +793,8 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -885,14 +807,12 @@ var _ = Describe("c3appfw test", func() { // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -908,8 +828,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -938,7 +857,7 @@ var _ = Describe("c3appfw test", func() { // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -955,15 +874,16 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -1007,13 +927,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -1023,14 +943,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1047,10 +961,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -1070,16 +985,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1095,14 +1004,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1170,7 +1080,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1180,9 +1090,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1193,8 +1103,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1212,14 +1122,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1238,10 +1142,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on S3 @@ -1274,16 +1179,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1305,15 +1204,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1379,7 +1279,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1389,8 +1289,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1400,8 +1300,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1419,14 +1319,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1454,7 +1348,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1464,7 +1359,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete apps on S3 @@ -1497,16 +1392,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1528,15 +1417,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1562,7 +1452,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1581,13 +1471,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1597,14 +1487,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1621,10 +1505,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1675,7 +1560,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1709,29 +1594,23 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1753,10 +1632,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -1784,16 +1664,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1866,14 +1740,15 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1921,13 +1796,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1936,14 +1811,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1960,10 +1829,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -1983,22 +1853,17 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -2049,14 +1914,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2122,7 +1988,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2132,8 +1998,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -2143,8 +2009,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2161,14 +2027,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2187,10 +2047,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on S3 @@ -2235,16 +2096,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2273,10 +2128,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2311,7 +2167,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2336,7 +2192,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2355,13 +2211,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2440,7 +2296,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2465,7 +2321,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2484,13 +2340,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2568,7 +2424,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2586,13 +2442,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2605,16 +2461,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2631,10 +2481,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2662,7 +2513,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2680,13 +2531,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2699,16 +2550,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2725,10 +2570,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2771,13 +2617,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2786,14 +2632,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2809,7 +2649,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] @@ -2821,7 +2662,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2875,7 +2716,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2892,13 +2733,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2934,16 +2775,10 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2958,12 +2793,13 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2983,7 +2819,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2992,7 +2828,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -3003,7 +2839,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -3012,14 +2848,14 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3030,14 +2866,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3054,7 +2884,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3081,7 +2912,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -3099,13 +2930,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3118,19 +2949,13 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3147,10 +2972,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -3185,13 +3011,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3210,18 +3036,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3278,11 +3100,11 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -3305,14 +3127,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3331,10 +3153,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -3342,10 +3164,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) // Deploy C3 SVA // Deploy the Cluster Manager @@ -3388,14 +3210,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3408,19 +3224,21 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // //############### UPGRADE APPS ################ // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster @@ -3430,7 +3248,7 @@ var _ = Describe("c3appfw test", func() { // uploadedApps = append(uploadedApps, uploadedFiles...) // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) + // testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) // // Ensure that the Cluster Manager goes to Ready phase // testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_aws/m4/appframework_aws_suite_test.go b/test/appframework_aws/m4/appframework_aws_suite_test.go index aa21c7084..44862712f 100644 --- a/test/appframework_aws/m4/appframework_aws_suite_test.go +++ b/test/appframework_aws/m4/appframework_aws_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,54 +39,17 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_aws/m4/appframework_aws_test.go b/test/appframework_aws/m4/appframework_aws_test.go index fca638acb..abd99f8fa 100644 --- a/test/appframework_aws/m4/appframework_aws_test.go +++ b/test/appframework_aws/m4/appframework_aws_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -54,15 +53,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).ToNot(HaveOccurred()) s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -71,30 +63,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,10 +107,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -171,11 +143,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -187,19 +159,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -233,10 +196,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# UPGRADE APPS ################ // Delete apps on S3 @@ -268,19 +232,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -305,15 +260,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + ClusterMasterBundleHash, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -354,10 +310,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -390,11 +346,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -403,19 +359,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -437,10 +384,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete V2 apps on S3 @@ -470,19 +418,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -504,15 +443,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + ClusterMasterBundleHash, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -567,11 +507,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -582,17 +522,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -605,7 +536,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) } // ############ Verify livenessProbe and readinessProbe config object and scripts############ @@ -623,10 +554,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -634,9 +566,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -654,8 +584,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -675,7 +604,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -687,7 +616,7 @@ var _ = Describe("m4appfw test", func() { searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -705,7 +634,8 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -720,13 +650,11 @@ var _ = Describe("m4appfw test", func() { shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -742,8 +670,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -753,9 +680,6 @@ var _ = Describe("m4appfw test", func() { err = deployment.UpdateCR(ctx, idxc) Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -767,7 +691,7 @@ var _ = Describe("m4appfw test", func() { // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -784,14 +708,15 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -833,29 +758,23 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 - indexersPerSite := 1 shReplicas := 3 + indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -869,10 +788,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -895,16 +815,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -917,15 +831,17 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { + /* Test Steps ################## SETUP #################### * Upload V1 apps to S3 for Monitoring Console @@ -968,10 +884,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1004,30 +920,21 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1049,12 +956,18 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) - // ############### UPGRADE APPS ################ + //############### UPGRADE APPS ################ + // Delete V2 apps on S3 + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) + + testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" @@ -1077,19 +990,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1163,7 +1067,7 @@ var _ = Describe("m4appfw test", func() { Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - // ############ VERIFY APPS UPDATED TO V2 ############# + //############# UPGRADE APPS ################ appVersion = "V2" cmAppSourceInfo.CrAppVersion = appVersion cmAppSourceInfo.CrAppList = appListV2 @@ -1175,15 +1079,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + ClusterMasterBundleHash, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps ################## SETUP #################### @@ -1228,11 +1133,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -1242,14 +1147,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1263,10 +1162,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -1289,20 +1189,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1358,14 +1253,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1431,7 +1327,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1441,9 +1337,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1453,8 +1349,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1473,17 +1369,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1502,10 +1389,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on S3 @@ -1551,19 +1439,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1589,10 +1468,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1626,7 +1506,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1651,7 +1531,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1670,20 +1550,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1763,7 +1643,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1788,7 +1668,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1807,13 +1687,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1821,7 +1701,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1870,7 +1750,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1892,7 +1772,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1908,11 +1788,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1920,25 +1800,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1955,14 +1826,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1984,7 +1856,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2000,11 +1872,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2012,25 +1884,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2047,14 +1910,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2090,11 +1954,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2102,19 +1966,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2130,7 +1985,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] @@ -2142,7 +1998,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) @@ -2166,7 +2022,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2208,11 +2064,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -2222,14 +2078,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2243,10 +2093,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") @@ -2256,8 +2107,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2267,8 +2117,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2326,7 +2175,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2344,14 +2194,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2383,7 +2234,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2398,11 +2249,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2410,7 +2261,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2439,7 +2290,7 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) @@ -2466,11 +2317,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2490,7 +2342,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2499,7 +2351,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2510,7 +2362,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2519,14 +2371,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -2537,14 +2389,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2558,11 +2404,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2584,7 +2431,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2600,11 +2447,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2612,28 +2459,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2650,14 +2488,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2684,11 +2523,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2696,7 +2535,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2709,14 +2548,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_aws/m4/manager_appframework_test.go b/test/appframework_aws/m4/manager_appframework_test.go index 03e41df85..7efc802a9 100644 --- a/test/appframework_aws/m4/manager_appframework_test.go +++ b/test/appframework_aws/m4/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -53,15 +52,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,30 +62,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -134,10 +106,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -170,11 +142,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -186,19 +158,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -232,10 +195,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# UPGRADE APPS ################ // Delete apps on S3 @@ -267,27 +231,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify Monitoring Console is Ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) // Get Pod age to check for pod resets later @@ -304,15 +256,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -353,10 +306,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -389,11 +342,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -402,19 +355,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -436,10 +380,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete V2 apps on S3 @@ -469,19 +414,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -503,15 +439,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -566,11 +503,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -581,17 +518,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -604,7 +532,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) } // ############ Verify livenessProbe and readinessProbe config object and scripts############ @@ -622,10 +550,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -633,9 +562,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -653,8 +580,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -674,7 +600,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -686,7 +612,7 @@ var _ = Describe("m4appfw test", func() { searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -704,7 +630,8 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -719,13 +646,11 @@ var _ = Describe("m4appfw test", func() { shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -741,8 +666,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -752,9 +676,6 @@ var _ = Describe("m4appfw test", func() { err = deployment.UpdateCR(ctx, idxc) Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -766,7 +687,7 @@ var _ = Describe("m4appfw test", func() { // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -783,14 +704,15 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -832,11 +754,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -847,14 +769,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -868,10 +784,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -894,16 +811,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -916,14 +827,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -967,10 +879,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1003,30 +915,22 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1048,10 +952,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ @@ -1076,19 +981,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1108,6 +1004,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Modify config map to trigger manual update") config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) + config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) err = deployment.UpdateCR(ctx, config) Expect(err).To(Succeed(), "Unable to update config map") @@ -1124,21 +1021,6 @@ var _ = Describe("m4appfw test", func() { config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) err = deployment.UpdateCR(ctx, config) @@ -1161,7 +1043,7 @@ var _ = Describe("m4appfw test", func() { Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - // ############ VERIFY APPS UPDATED TO V2 ############# + //############### UPGRADE APPS ################ appVersion = "V2" cmAppSourceInfo.CrAppVersion = appVersion cmAppSourceInfo.CrAppList = appListV2 @@ -1173,14 +1055,15 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1226,11 +1109,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1240,14 +1123,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1261,10 +1138,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on S3 @@ -1287,20 +1165,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1340,7 +1213,7 @@ var _ = Describe("m4appfw test", func() { // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# + // ########## Verify Manual Poll disabled after the poll is triggered ################# // Verify config map set back to off after poll trigger testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) @@ -1356,15 +1229,16 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, managerappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + It("integration, m4, managerappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps ################## SETUP #################### @@ -1429,7 +1303,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1439,9 +1313,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1451,8 +1325,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1471,17 +1345,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1500,10 +1365,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on S3 @@ -1549,19 +1415,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1587,288 +1444,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, managerappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to S3 for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to S3 for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from S3 during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to S3 for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from S3 - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to S3 for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to S3 for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to S3 for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to S3 for Deployer - testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for SearchHeadCluster to reach Ready phase - err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for SearchHeadCluster to reach Ready phase") - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to S3 for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to S3 for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from S3 during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to S3 for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from S3 - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to S3 for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to S3 for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to S3 for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to S3 for Deployer - testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1890,7 +1474,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1906,11 +1490,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1918,25 +1502,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1953,14 +1528,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1982,7 +1558,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1998,11 +1574,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2010,25 +1586,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2045,14 +1612,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2088,11 +1656,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2100,19 +1668,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2128,7 +1687,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] @@ -2140,7 +1700,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2164,7 +1724,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2206,11 +1766,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2220,14 +1780,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2241,10 +1795,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster") @@ -2254,8 +1809,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Manager CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Manager") // Set AppsRepoPollInterval for Cluster Manager to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") @@ -2265,8 +1819,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2324,7 +1877,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2342,14 +1896,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2381,7 +1936,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2396,11 +1951,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2408,7 +1963,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2437,7 +1992,7 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2464,11 +2019,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, managerappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2488,7 +2044,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2497,7 +2053,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2508,7 +2064,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2517,14 +2073,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2556,11 +2112,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2582,7 +2139,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2598,11 +2155,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2610,28 +2167,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2648,14 +2196,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2682,11 +2231,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2694,7 +2243,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2707,14 +2256,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_aws/s1/appframework_aws_suite_test.go b/test/appframework_aws/s1/appframework_aws_suite_test.go index 252889490..0a9f126ad 100644 --- a/test/appframework_aws/s1/appframework_aws_suite_test.go +++ b/test/appframework_aws/s1/appframework_aws_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -60,39 +46,10 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_aws/s1/appframework_aws_test.go b/test/appframework_aws/s1/appframework_aws_test.go index bf91e80ce..a412f216d 100644 --- a/test/appframework_aws/s1/appframework_aws_test.go +++ b/test/appframework_aws/s1/appframework_aws_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -47,42 +46,15 @@ var _ = Describe("s1appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -130,10 +102,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -164,9 +136,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -185,7 +157,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -211,7 +183,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# @@ -234,7 +207,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -253,7 +226,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -310,10 +284,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -334,9 +308,9 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -354,7 +328,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -371,7 +345,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############# DOWNGRADE APPS ################ // Delete apps on S3 @@ -396,7 +371,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -418,7 +393,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -481,10 +457,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -511,9 +487,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -531,7 +507,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -555,7 +531,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -566,8 +543,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -575,16 +551,17 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady) // Verify Monitoring Console is Ready and stays in ready state //testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) //########### SCALING UP VERIFICATION ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -599,24 +576,24 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) Expect(err).To(Succeed(), "Failed to scale down Standalone") // Ensure Standalone is scaling down - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingDown) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady) // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) //########### SCALING DOWN VERIFICATION ####### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -669,9 +646,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -686,7 +663,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -699,15 +676,15 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### SCALING UP ################## // Scale up Standalone instance testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -715,7 +692,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -736,7 +713,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -750,7 +727,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -773,7 +751,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to S3 @@ -782,9 +760,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, s3TestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -805,7 +783,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Ensure Standalone goes to Ready phase testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -818,7 +796,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# @@ -829,7 +808,7 @@ var _ = Describe("s1appfw test", func() { // Download ES App from S3 testcaseEnvInst.Log.Info("Download updated ES app from S3") - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to S3 for Standalone @@ -840,7 +819,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -853,7 +832,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = esApp standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -884,7 +864,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -894,9 +874,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -923,7 +903,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -974,10 +955,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1003,9 +984,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 0) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 0) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1023,7 +1004,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1038,7 +1019,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ @@ -1061,7 +1043,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Step: Wait for Phase Change") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) testcaseEnvInst.Log.Info("Step: Standalone Ready") @@ -1077,7 +1059,8 @@ var _ = Describe("s1appfw test", func() { // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1115,7 +1098,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1148,7 +1132,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download the extra apps from S3 for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 for first Standalone @@ -1166,9 +1150,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1180,10 +1164,10 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, s3TestDirStandalone2, 60) + appFrameworkSpecStandalone2 := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, s3TestDirStandalone2, 60) specStandalone2 := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1218,7 +1202,8 @@ var _ = Describe("s1appfw test", func() { standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1250,10 +1235,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1278,7 +1263,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1289,9 +1274,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1309,7 +1294,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1357,7 +1342,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1366,9 +1351,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1383,7 +1368,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1402,7 +1387,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1414,7 +1399,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1447,7 +1433,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1456,9 +1442,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1473,13 +1459,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1491,7 +1477,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1529,9 +1516,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1547,7 +1534,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1559,7 +1546,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on S3 bucket) appName := appListV1[0] @@ -1571,7 +1559,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileName) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1609,7 +1597,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to S3 App Source @@ -1640,9 +1628,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1659,7 +1647,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) Expect(err).To(Succeed(), "Unable to obtain secret object") @@ -1676,10 +1664,11 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1700,10 +1689,11 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Check no apps are updated as auth key is incorrect - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key to correct one########### // Apply spec with correct credentials @@ -1711,7 +1701,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to update secret Object") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1724,7 +1714,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1752,7 +1743,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to S3 for Standalone @@ -1761,9 +1752,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 120) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 120) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1778,7 +1769,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1797,7 +1788,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1810,7 +1801,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1835,7 +1827,7 @@ var _ = Describe("s1appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -1844,7 +1836,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 @@ -1856,9 +1848,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1886,7 +1878,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1911,7 +1904,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to S3 for Standalone @@ -1920,9 +1913,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1937,13 +1930,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") @@ -1958,7 +1951,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1991,10 +1985,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -2029,9 +2023,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2050,7 +2044,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/appframework_az/c3/appframework_azure_suite_test.go b/test/appframework_az/c3/appframework_azure_suite_test.go index 313c7c4fc..cea99c9b1 100644 --- a/test/appframework_az/c3/appframework_azure_suite_test.go +++ b/test/appframework_az/c3/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurec3appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -53,49 +39,17 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go index c4c6b4eff..9b29b4064 100644 --- a/test/appframework_az/c3/appframework_azure_test.go +++ b/test/appframework_az/c3/appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -50,42 +49,12 @@ var _ = Describe("c3appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -134,7 +103,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -171,13 +140,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -190,23 +159,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -225,10 +184,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on Azure @@ -262,24 +222,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -295,10 +246,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -345,10 +297,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -385,13 +337,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -404,23 +356,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -436,10 +378,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############## DOWNGRADE APPS ############### // Delete apps on Azure @@ -472,24 +415,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -505,10 +439,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -568,13 +503,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -584,14 +519,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -617,7 +546,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -627,13 +557,11 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -651,8 +579,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -674,12 +601,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -690,7 +612,7 @@ var _ = Describe("c3appfw test", func() { // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -708,19 +630,18 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -736,8 +657,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -762,7 +682,7 @@ var _ = Describe("c3appfw test", func() { // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -779,15 +699,16 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -831,13 +752,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -847,14 +768,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -871,10 +786,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -895,16 +811,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -920,14 +830,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -950,7 +861,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -977,10 +888,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -988,10 +899,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) // Deploy C3 SVA // Deploy the Cluster Master @@ -1034,14 +945,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1054,13 +959,15 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ // Delete ES app on Azure @@ -1071,7 +978,7 @@ var _ = Describe("c3appfw test", func() { // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1082,16 +989,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1104,11 +1005,12 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = esApp shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1176,7 +1078,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1186,9 +1088,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1199,8 +1101,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1218,14 +1120,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1253,7 +1149,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1263,7 +1160,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on Azure @@ -1297,16 +1194,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1328,15 +1219,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1405,7 +1297,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1415,8 +1307,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1426,8 +1318,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1445,14 +1337,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1475,10 +1361,11 @@ var _ = Describe("c3appfw test", func() { err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete apps on Azure @@ -1512,16 +1399,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1543,15 +1424,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1577,7 +1459,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1597,13 +1479,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1613,14 +1495,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1637,10 +1513,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1691,7 +1568,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1725,29 +1602,23 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1769,10 +1640,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -1801,16 +1673,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1883,14 +1749,15 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1938,13 +1805,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1953,14 +1820,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1977,10 +1838,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -2001,22 +1863,17 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -2067,14 +1924,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2140,7 +1998,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2150,8 +2008,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -2161,8 +2019,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2179,14 +2037,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2205,10 +2057,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on Azure @@ -2254,16 +2107,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2292,10 +2139,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2330,7 +2178,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2355,7 +2203,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2375,13 +2223,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2460,7 +2308,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2485,7 +2333,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2505,13 +2353,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2589,7 +2437,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2608,13 +2456,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2627,16 +2475,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2653,10 +2495,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2684,7 +2527,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2703,13 +2546,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2722,16 +2565,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2748,10 +2585,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2795,13 +2633,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2810,14 +2648,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2833,7 +2665,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] @@ -2846,7 +2679,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) @@ -2901,7 +2734,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2919,13 +2752,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2961,16 +2794,10 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2985,12 +2812,13 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -3011,7 +2839,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -3026,7 +2854,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) @@ -3035,14 +2863,14 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3053,14 +2881,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3077,7 +2899,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3104,7 +2927,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3123,13 +2946,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3142,19 +2965,13 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3171,10 +2988,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -3209,13 +3027,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3234,18 +3052,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3302,7 +3116,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) }) diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go index 8a8a1ec5c..e18500926 100644 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ b/test/appframework_az/c3/manager_appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -49,42 +48,12 @@ var _ = Describe("c3appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -133,7 +102,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -169,13 +138,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -188,23 +157,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -223,10 +182,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on Azure @@ -260,24 +220,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -293,10 +244,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -343,10 +295,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -383,13 +335,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -402,23 +354,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -434,10 +376,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############## DOWNGRADE APPS ############### // Delete apps on Azure @@ -470,24 +413,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -503,10 +437,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -566,13 +501,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -582,14 +517,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -615,7 +544,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -625,13 +555,11 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -649,8 +577,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -672,12 +599,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -688,7 +610,7 @@ var _ = Describe("c3appfw test", func() { // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -706,19 +628,18 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -734,8 +655,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -760,7 +680,7 @@ var _ = Describe("c3appfw test", func() { // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -777,15 +697,16 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -829,13 +750,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -845,14 +766,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -869,10 +784,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -893,16 +809,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -918,14 +828,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -948,7 +859,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -975,10 +886,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -986,10 +897,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) // Deploy C3 SVA // Deploy the Cluster Master @@ -1032,14 +943,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1052,13 +957,15 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ // Delete ES app on Azure @@ -1069,7 +976,7 @@ var _ = Describe("c3appfw test", func() { // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1080,16 +987,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - // // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1102,12 +1003,13 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = esApp shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1175,7 +1077,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1185,9 +1087,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1198,8 +1100,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1217,14 +1119,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1252,7 +1148,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1262,7 +1159,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on Azure @@ -1296,16 +1193,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1327,15 +1218,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1404,7 +1296,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1414,8 +1306,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1425,8 +1317,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1444,14 +1336,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1475,10 +1361,11 @@ var _ = Describe("c3appfw test", func() { err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete apps on Azure @@ -1512,16 +1399,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1543,15 +1424,16 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1577,7 +1459,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1597,13 +1479,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1613,14 +1495,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1637,10 +1513,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1691,7 +1568,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1725,29 +1602,23 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1769,10 +1640,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -1801,16 +1673,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1883,14 +1749,15 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1938,13 +1805,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1953,14 +1820,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1977,10 +1838,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -2001,22 +1863,17 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -2067,14 +1924,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2140,7 +1998,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2150,8 +2008,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -2161,8 +2019,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2179,14 +2037,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2205,10 +2057,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on Azure @@ -2254,16 +2107,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2292,10 +2139,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2330,7 +2178,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2355,7 +2203,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2375,13 +2223,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2460,7 +2308,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2485,7 +2333,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2505,13 +2353,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2589,7 +2437,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2608,13 +2456,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2627,16 +2475,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2653,10 +2495,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2684,7 +2527,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2703,13 +2546,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2722,16 +2565,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2748,10 +2585,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2795,13 +2633,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2810,14 +2648,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2833,7 +2665,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] @@ -2846,7 +2679,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2901,7 +2734,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2919,13 +2752,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2961,16 +2794,10 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2985,12 +2812,13 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -3011,7 +2839,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -3026,7 +2854,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) @@ -3035,14 +2863,14 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3053,14 +2881,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3077,7 +2899,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3104,7 +2927,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3123,13 +2946,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3142,19 +2965,13 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3171,10 +2988,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -3209,13 +3027,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3234,18 +3052,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3302,7 +3116,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) }) diff --git a/test/appframework_az/m4/appframework_azure_suite_test.go b/test/appframework_az/m4/appframework_azure_suite_test.go index 6184e543b..b54cf1770 100644 --- a/test/appframework_az/m4/appframework_azure_suite_test.go +++ b/test/appframework_az/m4/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurem4appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -53,53 +39,17 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_az/m4/appframework_azure_test.go b/test/appframework_az/m4/appframework_azure_test.go index ce5cba659..60826a38a 100644 --- a/test/appframework_az/m4/appframework_azure_test.go +++ b/test/appframework_az/m4/appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -53,15 +52,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).ToNot(HaveOccurred()) azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,31 +62,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,10 +106,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -171,11 +142,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -187,28 +158,15 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -224,10 +182,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# UPGRADE APPS ################ // Delete apps on Azure @@ -257,19 +216,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -294,15 +244,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -343,10 +294,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -379,11 +330,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -392,19 +343,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -426,10 +368,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete V2 apps on Azure @@ -460,19 +403,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -494,15 +428,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -557,11 +492,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -572,17 +507,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -595,7 +521,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) } //########### INITIAL VERIFICATIONS ######### @@ -607,16 +533,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -634,8 +559,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -655,7 +579,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -667,7 +591,7 @@ var _ = Describe("m4appfw test", func() { searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -685,20 +609,19 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -714,8 +637,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -739,7 +661,7 @@ var _ = Describe("m4appfw test", func() { // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -756,14 +678,15 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -805,11 +728,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -820,14 +743,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -841,10 +758,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -868,16 +786,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -890,14 +802,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -941,10 +854,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -977,30 +890,21 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1022,10 +926,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ @@ -1050,19 +955,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1147,14 +1043,15 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1200,11 +1097,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1214,14 +1111,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1235,10 +1126,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -1262,20 +1154,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1331,14 +1218,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1404,7 +1292,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1414,9 +1302,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1426,8 +1314,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1446,17 +1334,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1475,10 +1354,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on Azure @@ -1525,19 +1405,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1563,10 +1434,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1600,7 +1472,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1625,7 +1497,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1645,20 +1517,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1737,7 +1609,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1762,7 +1634,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1782,13 +1654,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1796,7 +1668,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1845,7 +1717,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1867,7 +1739,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1884,11 +1756,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1896,25 +1768,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1931,14 +1794,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1960,7 +1824,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1977,11 +1841,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1989,25 +1853,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.DeleteOperatorPod() - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2024,14 +1879,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2068,11 +1924,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2080,19 +1936,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2108,7 +1955,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] @@ -2121,7 +1969,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2146,7 +1994,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2188,11 +2036,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2202,14 +2050,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2223,10 +2065,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") @@ -2236,8 +2079,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2247,8 +2089,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2307,7 +2148,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2325,14 +2167,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2364,7 +2207,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2380,11 +2223,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2392,7 +2235,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2421,19 +2264,10 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2448,11 +2282,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2473,7 +2308,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2488,7 +2323,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") @@ -2497,14 +2332,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2515,14 +2350,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2536,11 +2365,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2562,7 +2392,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2579,11 +2409,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2591,28 +2421,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2629,14 +2450,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2663,11 +2485,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2675,7 +2497,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2688,14 +2510,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/m4/manager_appframework_azure_test.go b/test/appframework_az/m4/manager_appframework_azure_test.go index 96d24efc2..21b5317d1 100644 --- a/test/appframework_az/m4/manager_appframework_azure_test.go +++ b/test/appframework_az/m4/manager_appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -52,15 +51,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -69,28 +61,7 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { @@ -134,10 +105,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -170,11 +141,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -186,28 +157,15 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -223,10 +181,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# UPGRADE APPS ################ // Delete apps on Azure @@ -256,19 +215,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -293,10 +243,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -342,10 +293,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -378,11 +329,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -391,19 +342,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -425,10 +367,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete V2 apps on Azure @@ -459,19 +402,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -493,10 +427,11 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -556,11 +491,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -571,17 +506,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -594,7 +520,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) } //########### INITIAL VERIFICATIONS ######### @@ -606,16 +532,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -633,8 +558,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -654,7 +578,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -666,7 +590,7 @@ var _ = Describe("m4appfw test", func() { searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -684,20 +608,19 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -713,8 +636,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -738,7 +660,7 @@ var _ = Describe("m4appfw test", func() { // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -755,14 +677,15 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -804,11 +727,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -819,14 +742,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -840,10 +757,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -867,16 +785,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -889,14 +801,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -940,10 +853,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -976,30 +889,21 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1021,10 +925,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ @@ -1049,19 +954,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1146,14 +1042,15 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1199,11 +1096,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1213,14 +1110,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1234,10 +1125,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on Azure @@ -1261,20 +1153,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1330,14 +1217,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, managerappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1403,7 +1291,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1413,9 +1301,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1425,8 +1313,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1445,17 +1333,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1474,10 +1353,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on Azure @@ -1524,19 +1404,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1562,10 +1433,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1599,7 +1471,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1624,7 +1496,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1644,20 +1516,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1737,7 +1609,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1762,7 +1634,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1782,13 +1654,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1796,7 +1668,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1867,7 +1739,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1884,11 +1756,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1896,25 +1768,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1931,10 +1794,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1960,7 +1824,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1977,11 +1841,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1989,25 +1853,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2024,10 +1879,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2068,11 +1924,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2080,19 +1936,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2108,7 +1955,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] @@ -2121,7 +1969,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2146,7 +1994,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2188,11 +2036,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2202,14 +2050,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2223,10 +2065,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") @@ -2236,8 +2079,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2247,8 +2089,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2307,7 +2148,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2325,10 +2167,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2364,7 +2207,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2380,11 +2223,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2392,7 +2235,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2421,19 +2264,10 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2448,7 +2282,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2473,7 +2308,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2488,7 +2323,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") @@ -2497,14 +2332,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2515,14 +2350,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2536,7 +2365,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -2562,7 +2392,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2579,11 +2409,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2591,28 +2421,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2629,10 +2450,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2663,11 +2485,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2675,7 +2497,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2688,14 +2510,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/s1/appframework_azure_suite_test.go b/test/appframework_az/s1/appframework_azure_suite_test.go index d3c06c3c4..0fc5eebad 100644 --- a/test/appframework_az/s1/appframework_azure_suite_test.go +++ b/test/appframework_az/s1/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azures1appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -53,49 +39,17 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_az/s1/appframework_azure_test.go b/test/appframework_az/s1/appframework_azure_test.go index 92736ef92..6b5dd172f 100644 --- a/test/appframework_az/s1/appframework_azure_test.go +++ b/test/appframework_az/s1/appframework_azure_test.go @@ -20,7 +20,6 @@ import ( "strings" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -43,44 +42,15 @@ var _ = Describe("s1appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) azTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -131,10 +101,10 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) appFrameworkSpecMC.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -164,9 +134,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -185,7 +155,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -202,7 +172,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# @@ -225,7 +196,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -244,7 +215,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -300,10 +272,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -324,9 +296,9 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -344,7 +316,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -361,7 +333,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############# DOWNGRADE APPS ################ // Delete apps on Azure @@ -387,7 +360,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -409,7 +382,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -474,10 +448,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -505,9 +479,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -525,7 +499,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -543,15 +517,15 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### SCALING UP ################## // Scale up Standalone instance testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -559,40 +533,41 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady) // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) //########### SCALING UP VERIFICATION ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############## SCALING DOWN ################# // Scale down Standalone instance testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) Expect(err).To(Succeed(), "Failed to scale down Standalone") // Ensure Standalone is scaling down - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingDown) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady) // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) //########### SCALING DOWN VERIFICATION ####### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -646,9 +621,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -663,7 +638,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -676,15 +651,15 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### SCALING UP ################## // Scale up Standalone instance testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -692,7 +667,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -714,7 +689,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -728,7 +703,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -751,7 +727,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -762,9 +738,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -785,7 +761,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -795,7 +771,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# @@ -805,7 +782,7 @@ var _ = Describe("s1appfw test", func() { azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) // Download ES App from Azure - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -817,7 +794,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -830,7 +807,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = esApp standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -861,7 +839,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -872,9 +850,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -901,7 +879,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -952,10 +931,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -981,9 +960,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1001,7 +980,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1016,7 +995,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ @@ -1038,7 +1018,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1048,7 +1028,8 @@ var _ = Describe("s1appfw test", func() { // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1082,7 +1063,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1115,7 +1097,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download the extra apps from Azure for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1134,9 +1116,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1148,10 +1130,10 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) + appFrameworkSpecStandalone2 := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) specStandalone2 := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1185,7 +1167,8 @@ var _ = Describe("s1appfw test", func() { standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1218,10 +1201,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1246,7 +1229,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1258,9 +1241,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1278,7 +1261,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1326,7 +1309,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1337,9 +1320,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1354,13 +1337,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1372,7 +1355,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1397,7 +1381,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1408,9 +1392,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1425,13 +1409,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1443,7 +1427,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1482,9 +1467,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1500,7 +1485,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1512,7 +1497,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) appName := appListV1[0] @@ -1525,7 +1511,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on Azure") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileName) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1564,7 +1550,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to Azure App Source @@ -1595,9 +1581,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1614,7 +1600,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, _ := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) secretData := secretStruct.Data @@ -1630,10 +1616,11 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1655,10 +1642,11 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Check no apps are updated as auth key is incorrect - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key to correct one########### // Apply spec with correct credentials @@ -1666,7 +1654,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to update secret Object") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1679,7 +1667,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1707,7 +1696,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1717,9 +1706,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1734,7 +1723,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1759,7 +1748,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1769,7 +1758,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1795,7 +1785,7 @@ var _ = Describe("s1appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -1811,9 +1801,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1841,7 +1831,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1866,7 +1857,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big app") @@ -1876,9 +1867,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1893,13 +1884,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") @@ -1914,7 +1905,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1947,10 +1939,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1985,9 +1977,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2006,7 +1998,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/appframework_gcp/c3/appframework_gcs_suite_test.go b/test/appframework_gcp/c3/appframework_gcs_suite_test.go index 9aa061bad..994369326 100644 --- a/test/appframework_gcp/c3/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -60,43 +46,10 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to Gcs - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Gcs after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files.") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files.") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go index 3a189f7a0..bbf81dd9a 100644 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_test.go @@ -17,7 +17,6 @@ import ( "context" //"encoding/json" "fmt" - "path/filepath" //"strings" //"time" @@ -26,7 +25,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -54,44 +52,13 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 5000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master", testenv.WithTimeout(5000)) + Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -139,7 +106,7 @@ var _ = Describe("c3appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -175,13 +142,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -194,23 +161,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -238,10 +195,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on GCS @@ -273,24 +231,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -306,10 +255,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -356,10 +306,10 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -396,13 +346,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -415,23 +365,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -447,10 +387,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############## DOWNGRADE APPS ############### // Delete apps on GCS @@ -482,24 +423,15 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -515,14 +447,15 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -566,13 +499,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -582,14 +515,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -606,10 +533,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on GCS @@ -629,16 +557,10 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -654,10 +576,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -691,13 +614,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -716,11 +639,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_gcp/c3/manager_appframework_test.go b/test/appframework_gcp/c3/manager_appframework_test.go index 7c10a6b87..25e5c9e51 100644 --- a/test/appframework_gcp/c3/manager_appframework_test.go +++ b/test/appframework_gcp/c3/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,45 +50,13 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - testenv.SpecifiedTestTimeout = 100000 + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "", testenv.WithTimeout(100000)) + Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Gcs - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -97,10 +64,10 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -112,7 +79,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for Monitoring Console and C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -125,19 +92,19 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -159,27 +126,27 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -236,42 +203,43 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Upload V2 apps to Gcs for Indexer Cluster + // Upload V2 apps to GCS for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V2 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -304,10 +272,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -322,7 +291,7 @@ var _ = Describe("c3appfw test", func() { switch testenv.ClusterProvider { case "eks": licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from Gcs") + Expect(err).To(Succeed(), "Unable to download license file from GCS") // Create License Config Map testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) case "azure": @@ -337,10 +306,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) default: fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with provider set as %v", testenv.ClusterProvider)) } - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console oldImage := "Refer to RELATED_SPLUNK_IMAGE_ENTERPRISE" newImage := "splunk/splunk:latest" @@ -402,7 +371,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE IMAGE ################ @@ -464,10 +433,10 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V2 apps to Gcs for Monitoring Console + * Upload V2 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster * Create app source for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ########### INITIAL VERIFICATIONS ########### @@ -479,7 +448,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods ############## DOWNGRADE APPS ############### - * Upload V1 apps on Gcs + * Upload V1 apps on GCS * Wait for Monitoring Console and C3 pods to be ready ########### FINAL VERIFICATIONS ############# * Verify Apps are Downloaded in App Deployment Info @@ -492,19 +461,19 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V2 apps to Gcs for Monitoring Console + // Upload V2 apps to GCS for Monitoring Console appVersion := "V2" appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -527,27 +496,27 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V2 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V2 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -592,42 +561,43 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############## DOWNGRADE APPS ############### - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexers", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexers", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V1 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -660,10 +630,11 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -672,7 +643,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps on Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps on GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app config and wait for pods to be ready ########## INITIAL VERIFICATIONS ############ @@ -707,29 +678,29 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP ################## - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -778,7 +749,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -788,7 +760,7 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -796,9 +768,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -816,8 +786,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -839,12 +808,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -855,7 +819,7 @@ var _ = Describe("c3appfw test", func() { // Search for data on newly added indexer searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -873,7 +837,8 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -886,14 +851,12 @@ var _ = Describe("c3appfw test", func() { // Verify no pods reset by checking the pod age shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -909,8 +872,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -935,7 +897,7 @@ var _ = Describe("c3appfw test", func() { // Search for data from removed indexer searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result @@ -952,20 +914,21 @@ var _ = Describe("c3appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############# INITIAL VERIFICATIONS ########## @@ -988,29 +951,29 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -1044,30 +1007,31 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs + // Upload V2 apps to GCS appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) appFileList = testenv.GetAppFileList(appListV2) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1092,20 +1056,21 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -1117,7 +1082,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -1135,39 +1100,39 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1177,9 +1142,9 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1190,8 +1155,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1235,14 +1200,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -1250,28 +1216,28 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) + // Upload appListLocal list of V2 apps to GCS (to be used for local install) appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1302,21 +1268,22 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -1328,7 +1295,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### Downgrade APPS ################ - * Upload V1 apps on Gcs + * Upload V1 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -1346,37 +1313,37 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV2[len(appListV2)/2:] appListCluster := appListV2[:len(appListV2)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Idxc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Shc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1386,8 +1353,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1397,8 +1364,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1451,7 +1418,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) } } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) for _, appSource := range allAppSourceInfo { if appSource.CrAppScope == enterpriseApi.ScopeCluster { @@ -1461,11 +1429,11 @@ var _ = Describe("c3appfw test", func() { } // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -1473,28 +1441,28 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V1 apps to Gcs (to be used for local install) + // Upload appListLocal list of V1 apps to GCS (to be used for local install) appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1525,21 +1493,22 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps ################## SETUP #################### * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Gcs for this test + * Add more apps than usual on GCS for this test * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############### VERIFICATIONS ############### * Verify Apps are Downloaded in App Deployment Info @@ -1557,34 +1526,34 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appList) appVersion := "V1" - // Download apps from Gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from Gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + // Download apps from GCS + testcaseEnvInst.Log.Info("Download bigger amount of apps from GCS for this test") + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files appList = append(appListV1, appList...) appFileList = testenv.GetAppFileList(appList) - // Upload app to Gcs for Indexer Cluster + // Upload app to GCS for Indexer Cluster gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Indexer Cluster") + Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload app to Gcs for Search Head Cluster + // Upload app to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1618,10 +1587,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1629,10 +1599,10 @@ var _ = Describe("c3appfw test", func() { It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled for manual update", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ########## INITIAL VERIFICATION ############# @@ -1660,19 +1630,19 @@ var _ = Describe("c3appfw test", func() { */ // ################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1694,32 +1664,32 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1750,38 +1720,39 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs for C3 + // Upload V2 apps to GCS for C3 appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to GCS", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V2 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1863,19 +1834,20 @@ var _ = Describe("c3appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############# INITIAL VERIFICATION ########## @@ -1902,29 +1874,29 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1957,30 +1929,31 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs + // Upload V2 apps to GCS appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) appFileList = testenv.GetAppFileList(appListV2) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1995,7 +1968,8 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyRFSFMet(ctx, deployment) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ testcaseEnvInst.Log.Info("Get config map for triggering manual update") @@ -2046,20 +2020,21 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -2071,7 +2046,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -2089,37 +2064,37 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Idxc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Shc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2129,8 +2104,8 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) + // Create App Framework Spec for Cluster manager with scope local and append cluster scope + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -2140,8 +2115,8 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -2184,14 +2159,15 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -2199,24 +2175,24 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) + // Upload appListLocal list of V2 apps to GCS (to be used for local install) appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // ############ ENABLE MANUAL POLL ############ @@ -2232,7 +2208,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2270,10 +2246,11 @@ var _ = Describe("c3appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2282,33 +2259,33 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework ############## VERIFICATIONS ################ * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install + * Upload more apps from GCS during bigger app install * Wait for polling interval to pass * Verify all apps are installed on Cluster Manager and Deployer */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2330,35 +2307,35 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big-size app to Gcs for Cluster Manager + // Upload big-size app to GCS for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Cluster Manager") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") + // Upload big-size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2369,18 +2346,18 @@ var _ = Describe("c3appfw test", func() { // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - // Upload more apps to Gcs for Cluster Manager + // Upload more apps to GCS for Cluster Manager appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload more apps to GCS for Cluster Manager") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") + // Upload more apps to GCS for Deployer + testcaseEnvInst.Log.Info("Upload more apps to GCS for Deployer") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase @@ -2411,33 +2388,33 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ############## VERIFICATIONS ################ * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install + * Upload more apps from GCS during bigger app install * Wait for polling interval to pass * Verify all apps are installed on Cluster Manager and Deployer */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2459,35 +2436,35 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big-size app to Gcs for Cluster Manager + // Upload big-size app to GCS for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Cluster Manager") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") + // Upload big-size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2498,18 +2475,18 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - // Upload more apps to Gcs for Cluster Manager + // Upload more apps to GCS for Cluster Manager appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload more apps to GCS for Cluster Manager") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") + // Upload more apps to GCS for Deployer + testcaseEnvInst.Log.Info("Upload more apps to GCS for Deployer") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase @@ -2547,7 +2524,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app install is in progress, restart the operator @@ -2562,34 +2539,34 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2602,7 +2579,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2628,10 +2605,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2641,7 +2619,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app download is in progress, restart the operator @@ -2656,34 +2634,34 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2696,7 +2674,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2722,10 +2700,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -2735,7 +2714,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### VERIFICATIONS ############# @@ -2747,34 +2726,34 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods * Disable the app - * Delete the app from Gcs + * Delete the app from GCS * Check for repo state in App Deployment Info */ //################## SETUP #################### appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2806,9 +2785,10 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") - // Verify repo state on App to be disabled to be 1 (i.e app present on Gcs bucket) + // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) @@ -2818,7 +2798,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on GCS") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2829,12 +2809,12 @@ var _ = Describe("c3appfw test", func() { // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - // Delete the file from Gcs + // Delete the file from GCS gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Gcs test directory", appFileName[0])) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCS test directory", appFileName[0])) - // Verify repo state is set to 2 (i.e app deleted from Gcs bucket) + // Verify repo state is set to 2 (i.e app deleted from GCS bucket) testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) }) @@ -2845,7 +2825,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app download is completed, upload new versions of the apps @@ -2868,34 +2848,34 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2907,19 +2887,19 @@ var _ = Describe("c3appfw test", func() { // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - // Upload V2 apps to Gcs for Indexer Cluster + // Upload V2 apps to GCS for Indexer Cluster appVersion = "V2" appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Get Pod age to check for pod resets later @@ -2931,7 +2911,7 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2955,17 +2935,18 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps ################## SETUP #################### - * Upload 15 apps of 100MB size each to Gcs for Indexer Cluster and Search Head Cluster for cluster scope + * Upload 15 apps of 100MB size each to GCS for Indexer Cluster and Search Head Cluster for cluster scope * Create app sources for Cluster Master and Deployer with cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -2980,7 +2961,7 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2989,34 +2970,34 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") - // Upload apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search head Cluster", appVersion)) + // Upload apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3051,7 +3032,8 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -3060,7 +3042,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * When app download is complete, delete apps from app directory @@ -3075,34 +3057,34 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download big size apps from Gcs + // Download big size apps from GCS appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big size app to Gcs for Indexer Cluster + // Upload big size app to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Indexer Cluster") + testcaseEnvInst.Log.Info("Upload big size app to GCS for Indexer Cluster") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Indexer Cluster") + Expect(err).To(Succeed(), "Unable to upload big size to GCS test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Search Head Cluster") + // Upload big size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big size to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3115,7 +3097,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") @@ -3144,10 +3126,11 @@ var _ = Describe("c3appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -3158,7 +3141,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP ################## - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Prepare and deploy C3 CRD with app framework * Verify IsDeploymentInProgress is set * Wait for the pods to be ready @@ -3168,27 +3151,27 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3218,7 +3201,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3275,16 +3258,16 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps ################## SETUP #################### - * Upload ES app to Gcs + * Upload ES app to GCS * Upload TA add-on app to location for Indexer cluster * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA * Prepare and deploy C3 CRD with app framework and wait for pods to be ready @@ -3292,46 +3275,46 @@ var _ = Describe("c3appfw test", func() { * Verify ES app is installed on Deployer and on Search Heads * Verify TA add-on app is installed on indexers ################## UPGRADE VERIFICATION ############# - * Update ES app on Gcs location + * Update ES app on GCS location * Verify updated ES app is installed on Deployer and on Search Heads */ //################## SETUP #################### - // Download ES app from Gcs + // Download ES app from GCS appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Gcs") + testcaseEnvInst.Log.Info("Download ES app from GCS") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download ES app file from GCS") - // Download Technology add-on app from Gcs - testcaseEnvInst.Log.Info("Download Technology add-on app from Gcs") + // Download Technology add-on app from GCS + testcaseEnvInst.Log.Info("Download Technology add-on app from GCS") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) + Expect(err).To(Succeed(), "Unable to download ES app file from GCS") - // Create directory for file upload to Gcs + // Create directory for file upload to GCS gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - // Upload ES app to Gcs - testcaseEnvInst.Log.Info("Upload ES app to Gcs") + // Upload ES app to GCS + testcaseEnvInst.Log.Info("Upload ES app to GCS") uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to Gcs test directory") + Expect(err).To(Succeed(), "Unable to upload ES app to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload Technology add-on apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to Gcs for Indexer Cluster", appVersion)) + // Upload Technology add-on apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileListIdxc, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -3339,10 +3322,10 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) // Deploy C3 SVA // Deploy the Cluster Manager @@ -3405,29 +3388,31 @@ var _ = Describe("c3appfw test", func() { deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // //############### UPGRADE APPS ################ - // // Download ES App from Gcs + // // Download ES App from GCS // appVersion = "V2" - // testcaseEnvInst.Log.Info("Download updated ES app from Gcs") + // testcaseEnvInst.Log.Info("Download updated ES app from GCS") // err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") - // // Upload V2 ES app to Gcs for Search Head Cluster - // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Gcs for Search Head Cluster", appVersion)) + // // Upload V2 ES app to GCS for Search Head Cluster + // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to GCS for Search Head Cluster", appVersion)) // uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Gcs test directory for Search Head Cluster", appVersion)) + // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to GCS test directory for Search Head Cluster", appVersion)) // uploadedApps = append(uploadedApps, uploadedFiles...) // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) + // testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) // // Ensure that the Cluster Manager goes to Ready phase // testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_gcp/m4/appframework_gcs_suite_test.go b/test/appframework_gcp/m4/appframework_gcs_suite_test.go index 8f4a28249..f70cb5a9b 100644 --- a/test/appframework_gcp/m4/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -60,44 +46,10 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - testenvInstance.Log.Info("logging download details", "bucket", testDataGcsBucket, "gcsAppDirV1", gcsAppDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_gcp/m4/appframework_gcs_test.go b/test/appframework_gcp/m4/appframework_gcs_test.go index 96759b180..6b64dd896 100644 --- a/test/appframework_gcp/m4/appframework_gcs_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -54,15 +53,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).ToNot(HaveOccurred()) gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -71,30 +63,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,10 +107,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -171,11 +143,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -187,19 +159,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -233,10 +196,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# UPGRADE APPS ################ // Delete apps on GCP @@ -268,19 +232,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -305,15 +260,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -354,10 +310,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -390,11 +346,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -403,19 +359,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -437,10 +384,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete V2 apps on GCP @@ -470,19 +418,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -504,15 +443,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -567,11 +507,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -582,17 +522,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -605,7 +536,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) } // ############ Verify livenessProbe and readinessProbe config object and scripts############ @@ -623,10 +554,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -634,9 +566,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -654,8 +584,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -675,7 +604,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -687,7 +616,7 @@ var _ = Describe("m4appfw test", func() { searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -705,7 +634,8 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -720,13 +650,11 @@ var _ = Describe("m4appfw test", func() { shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -742,8 +670,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -767,7 +694,7 @@ var _ = Describe("m4appfw test", func() { // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -784,14 +711,15 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -833,11 +761,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -848,14 +776,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -869,10 +791,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on GCP @@ -895,16 +818,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -917,14 +834,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -968,10 +886,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1004,30 +922,21 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1049,10 +958,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ @@ -1077,19 +987,10 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1175,14 +1076,15 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1228,11 +1130,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -1242,14 +1144,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1263,10 +1159,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on GCP @@ -1289,20 +1186,15 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1358,14 +1250,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1431,7 +1324,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1441,9 +1334,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1453,8 +1346,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1473,17 +1366,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1502,10 +1386,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + ClusterMasterBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on GCP @@ -1551,19 +1436,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1589,10 +1465,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1626,7 +1503,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1651,7 +1528,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1670,20 +1547,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1763,7 +1640,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1788,7 +1665,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1807,13 +1684,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1821,7 +1698,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1870,7 +1747,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1892,7 +1769,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1908,11 +1785,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1920,25 +1797,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1955,14 +1823,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1984,7 +1853,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -2000,11 +1869,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2012,25 +1881,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.DeleteOperatorPod() - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2047,14 +1907,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2090,11 +1951,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2102,19 +1963,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2130,7 +1982,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) appName := appListV1[0] @@ -2142,7 +1995,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on GCS") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) @@ -2166,7 +2019,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2208,11 +2061,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -2222,14 +2075,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2243,10 +2090,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") @@ -2256,8 +2104,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2267,8 +2114,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2326,7 +2172,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2344,14 +2191,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2383,7 +2231,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2398,11 +2246,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2410,7 +2258,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2439,7 +2287,7 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) @@ -2466,11 +2314,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2490,7 +2339,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2499,7 +2348,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2510,7 +2359,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) gcsTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) @@ -2519,14 +2368,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -2537,14 +2386,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2558,11 +2401,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2584,7 +2428,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2600,11 +2444,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2612,28 +2456,19 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2650,14 +2485,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2684,11 +2520,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2696,7 +2532,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2709,14 +2545,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_gcp/m4/manager_appframework_test.go b/test/appframework_gcp/m4/manager_appframework_test.go index bc6837431..05e07be5b 100644 --- a/test/appframework_gcp/m4/manager_appframework_test.go +++ b/test/appframework_gcp/m4/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -53,15 +52,8 @@ var _ = Describe("m4appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,30 +62,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -134,10 +106,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -170,11 +142,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) @@ -186,7 +158,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -232,10 +204,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# UPGRADE APPS ################ // Delete apps on GCP @@ -267,7 +240,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -304,15 +277,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -353,10 +327,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -389,11 +363,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -402,7 +376,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -436,10 +410,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ // Delete V2 apps on GCP @@ -469,7 +444,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -503,15 +478,16 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -566,11 +542,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -604,7 +580,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) } // ############ Verify livenessProbe and readinessProbe config object and scripts############ @@ -622,10 +598,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -633,9 +610,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -653,8 +628,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -674,7 +648,7 @@ var _ = Describe("m4appfw test", func() { podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, "main", podName) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -686,7 +660,7 @@ var _ = Describe("m4appfw test", func() { searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -704,7 +678,8 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -719,13 +694,11 @@ var _ = Describe("m4appfw test", func() { shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -741,8 +714,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -766,7 +738,7 @@ var _ = Describe("m4appfw test", func() { // Search for data from removed indexer searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) + searchResultsResp, err = testenv.PerformSearchSync(ctx, deployment, searchPod, searchString) Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) // Verify result. @@ -783,14 +755,15 @@ var _ = Describe("m4appfw test", func() { Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, shcPodNames) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -832,11 +805,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -868,10 +841,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on GCP @@ -894,7 +868,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -916,14 +890,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -967,10 +942,10 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1003,18 +978,18 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) siteCount := 3 shReplicas := 3 indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1048,10 +1023,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ @@ -1076,7 +1052,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1174,14 +1150,15 @@ var _ = Describe("m4appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1227,11 +1204,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1262,10 +1239,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete V1 apps on GCP @@ -1288,7 +1266,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1301,7 +1279,8 @@ var _ = Describe("m4appfw test", func() { // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1357,14 +1336,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1430,7 +1410,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1440,9 +1420,9 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1452,8 +1432,8 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1501,10 +1481,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + clusterManagerBundleHash, err := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) //############### UPGRADE APPS ################ // Delete apps on GCP @@ -1550,7 +1531,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to update config map") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1588,10 +1569,11 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfoCluster.CrAppList = appListCluster shcAppSourceInfoCluster.CrAppFileList = clusterappFileList allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) @@ -1625,7 +1607,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1650,7 +1632,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1669,20 +1651,20 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1761,7 +1743,7 @@ var _ = Describe("m4appfw test", func() { // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1786,7 +1768,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1805,13 +1787,13 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1819,7 +1801,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1868,7 +1850,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1890,7 +1872,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1906,11 +1888,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1918,13 +1900,13 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1953,14 +1935,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1982,7 +1965,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1998,11 +1981,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2010,13 +1993,13 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2045,14 +2028,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2088,11 +2072,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2100,7 +2084,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2128,7 +2112,8 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) appName := appListV1[0] @@ -2140,7 +2125,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on GCS") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2164,7 +2149,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2206,11 +2191,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2241,10 +2226,11 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) // Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster") @@ -2254,8 +2240,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Manager CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Manager") // Set AppsRepoPollInterval for Cluster Manager to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") @@ -2265,8 +2250,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2324,7 +2308,8 @@ var _ = Describe("m4appfw test", func() { //########## UPGRADE VERIFICATIONS ############ testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Wait for app repo state to change, indicating poll interval has completed testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") @@ -2342,14 +2327,15 @@ var _ = Describe("m4appfw test", func() { shcAppSourceInfo.CrAppList = appListV2 shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2381,7 +2367,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2396,11 +2382,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2408,7 +2394,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2437,7 +2423,7 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2464,11 +2450,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2488,7 +2475,7 @@ var _ = Describe("m4appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -2497,7 +2484,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2508,7 +2495,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) @@ -2517,14 +2504,14 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2556,11 +2543,12 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2582,7 +2570,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2598,11 +2586,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2610,13 +2598,13 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") @@ -2648,14 +2636,15 @@ var _ = Describe("m4appfw test", func() { cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) + testcaseEnvInst.VerifyNoPodResetByUID(ctx, splunkPodUIDs, nil) }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2682,11 +2671,11 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecShc := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2694,7 +2683,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") diff --git a/test/appframework_gcp/s1/appframework_gcs_suite_test.go b/test/appframework_gcp/s1/appframework_gcs_suite_test.go index af2fab4c2..74de1be0d 100644 --- a/test/appframework_gcp/s1/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGCSBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -60,39 +46,10 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + testenvInstance, appListV1, appListV2, err = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + Expect(testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2)).To(Succeed()) }) diff --git a/test/appframework_gcp/s1/appframework_gcs_test.go b/test/appframework_gcp/s1/appframework_gcs_test.go index edfd8019f..5e4109507 100644 --- a/test/appframework_gcp/s1/appframework_gcs_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -47,42 +46,15 @@ var _ = Describe("s1appfw test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).ToNot(HaveOccurred()) gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + Expect(testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGCSBucket, uploadedApps), filePresentOnOperator)).To(Succeed()) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -130,10 +102,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -164,9 +136,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -185,7 +157,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -211,7 +183,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# @@ -234,7 +207,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -253,7 +226,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV2 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -310,10 +284,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -334,9 +308,9 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -354,7 +328,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -371,7 +345,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############# DOWNGRADE APPS ################ // Delete apps on gcs @@ -396,7 +371,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -418,7 +393,8 @@ var _ = Describe("s1appfw test", func() { mcAppSourceInfo.CrAppList = appListV1 mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -481,10 +457,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -511,9 +487,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -531,7 +507,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -555,7 +531,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //Delete configMap Object err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) @@ -566,8 +543,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -575,16 +551,17 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady) // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) //########### SCALING UP VERIFICATION ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -599,24 +576,24 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) Expect(err).To(Succeed(), "Failed to scale down Standalone") // Ensure Standalone is scaling down - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingDown) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady) // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) //########### SCALING DOWN VERIFICATION ####### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -669,9 +646,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -686,7 +663,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -699,15 +676,15 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### SCALING UP ################## // Scale up Standalone instance testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -715,7 +692,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Failed to scale up Standalone") // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -736,7 +713,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -750,7 +727,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -773,7 +751,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from gcs") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to gcs @@ -782,9 +760,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -805,7 +783,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Ensure Standalone goes to Ready phase testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -818,7 +796,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############## UPGRADE APPS ################# @@ -829,7 +808,7 @@ var _ = Describe("s1appfw test", func() { // Download ES App from gcs testcaseEnvInst.Log.Info("Download updated ES app from gcs") - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to gcs for Standalone @@ -840,7 +819,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -853,7 +832,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = esApp standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -884,7 +864,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from gcs testcaseEnvInst.Log.Info("Download bigger amount of apps from gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -894,9 +874,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -923,7 +903,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -974,10 +955,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1003,9 +984,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1023,7 +1004,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1038,7 +1019,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") //############### UPGRADE APPS ################ @@ -1059,7 +1041,7 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1069,7 +1051,8 @@ var _ = Describe("s1appfw test", func() { // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ ENABLE MANUAL POLL ############ appVersion = "V2" @@ -1103,7 +1086,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1136,7 +1120,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from GCS testcaseEnvInst.Log.Info("Download the extra apps from GCS for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to GCS for first Standalone @@ -1154,9 +1138,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1168,10 +1152,10 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) + appFrameworkSpecStandalone2 := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) specStandalone2 := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1206,7 +1190,8 @@ var _ = Describe("s1appfw test", func() { standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1238,10 +1223,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1266,7 +1251,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1277,9 +1262,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1297,7 +1282,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1345,7 +1330,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1354,9 +1339,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1371,7 +1356,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1390,7 +1375,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1402,7 +1387,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1435,7 +1421,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1444,9 +1430,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1461,13 +1447,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) + testcaseEnvInst.DeleteOperatorPod() // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1479,7 +1465,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1517,9 +1504,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1535,7 +1522,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1547,7 +1534,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) appName := appListV1[0] @@ -1559,7 +1547,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to disable apps on GCS") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileName) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1597,7 +1585,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to GCS App Source @@ -1628,9 +1616,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1647,7 +1635,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) Expect(err).To(Succeed(), "Unable to obtain secret object") @@ -1664,10 +1652,11 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1688,10 +1677,11 @@ var _ = Describe("s1appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Check no apps are updated as auth key is incorrect - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // ############ Modify secret key to correct one########### // Apply spec with correct credentials @@ -1699,7 +1689,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to update secret Object") // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1712,7 +1702,8 @@ var _ = Describe("s1appfw test", func() { standaloneAppSourceInfo.CrAppList = appListV2 standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1740,7 +1731,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to GCS for Standalone @@ -1749,9 +1740,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1766,7 +1757,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1785,7 +1776,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1798,7 +1789,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1823,7 +1815,7 @@ var _ = Describe("s1appfw test", func() { //################## SETUP #################### // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") Expect(err).To(Succeed(), "Unable to create file on operator") filePresentOnOperator = true @@ -1832,7 +1824,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCS @@ -1844,9 +1836,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1874,7 +1866,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1899,7 +1892,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to GCS for Standalone @@ -1908,9 +1901,9 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1925,13 +1918,13 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) + opPod := testcaseEnvInst.GetOperatorPodName() podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") @@ -1946,7 +1939,8 @@ var _ = Describe("s1appfw test", func() { standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") }) }) @@ -1979,10 +1973,10 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) + appFrameworkSpecMC := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -2017,9 +2011,9 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2038,7 +2032,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/custom_resource_crud/crud_test_shared.go b/test/custom_resource_crud/crud_test_shared.go new file mode 100644 index 000000000..2ef0de333 --- /dev/null +++ b/test/custom_resource_crud/crud_test_shared.go @@ -0,0 +1,258 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunS1CPUUpdateTest runs the standard S1 CPU limit update test workflow +func RunS1CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string, newCPULimits string) { + // Deploy and verify Standalone + standalone, err := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") + + // Verify telemetry + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + Expect(testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime)).To(Succeed()) + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Verify CPU limits before updating the CR + standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + Expect(testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits)).To(Succeed()) + + // Change CPU limits to trigger CR update + standalone.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + err = deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") + + // Verify Standalone is updating + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseUpdating)).To(Succeed()) + + // Verify Standalone goes to ready state + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseReady)).To(Succeed()) + + // Verify Monitoring Console is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed()) + + // Verify CPU limits after updating the CR + Expect(testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits)).To(Succeed()) +} + +// RunC3CPUUpdateTest runs the standard C3 CPU limit update test workflow +func RunC3CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + Expect(config.DeployAndVerifyC3(ctx, deployment, testcaseEnvInst, 3, true /*shc*/, mcRef)).To(Succeed(), "Unable to deploy C3 cluster") + + // Verify telemetry + Expect(testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime)).To(Succeed()) + + // Deploy and verify Monitoring Console, RF/SF + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + Expect(testcaseEnvInst.StandardC3Verification(ctx, deployment, mc)).To(Succeed()) + + // Verify CPU limits on Indexers before updating the CR + indexerCount := 3 + Expect(testcaseEnvInst.VerifyIndexerCPULimits(deployment, indexerCount, defaultCPULimits)).To(Succeed()) + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) + Expect(testenv.GetInstanceWithExpect(ctx, deployment, idxc, instanceName, "Unable to get instance of Indexer Cluster")).To(Succeed()) + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + Expect(testenv.UpdateCRWithExpect(ctx, deployment, idxc, "Unable to deploy Indexer Cluster with updated CR")).To(Succeed()) + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-idxc" + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName)).To(Succeed()) + + // Verify Indexers go to ready state + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed()) + + // Verify CPU limits on Indexers after updating the CR + Expect(testcaseEnvInst.VerifyIndexerCPULimits(deployment, indexerCount, newCPULimits)).To(Succeed()) + + // Verify CPU limits on Search Heads before updating the CR + searchHeadCount := 3 + Expect(testcaseEnvInst.VerifySearchHeadCPULimits(deployment, searchHeadCount, defaultCPULimits)).To(Succeed()) + + // Change CPU limits to trigger CR update + shc := &enterpriseApi.SearchHeadCluster{} + instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) + Expect(testenv.GetInstanceWithExpect(ctx, deployment, shc, instanceName, "Unable to fetch Search Head Cluster deployment")).To(Succeed()) + + shc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + Expect(testenv.UpdateCRWithExpect(ctx, deployment, shc, "Unable to deploy Search Head Cluster with updated CR")).To(Succeed()) + + // Verify Search Head Cluster is updating + Expect(testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating)).To(Succeed()) + + // Verify Search Head go to ready state + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed()) + + // Verify Monitoring Console is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed()) + + // Verify CPU limits on Search Heads after updating the CR + Expect(testcaseEnvInst.VerifySearchHeadCPULimits(deployment, searchHeadCount, newCPULimits)).To(Succeed()) +} + +// RunC3PVCDeletionTest runs the standard C3 PVC deletion test workflow +func RunC3PVCDeletionTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, verificationTimeout time.Duration) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + Expect(config.DeployAndVerifyC3(ctx, deployment, testcaseEnvInst, 3, true /*shc*/, mcRef)).To(Succeed(), "Unable to deploy C3 cluster") + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed()) + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + clusterManagerType := config.ClusterManagerPVCType() + Expect(testenv.VerifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, true, verificationTimeout)).To(Succeed()) + + // Delete the Search Head Cluster + Expect(testenv.GetAndDeleteCR(ctx, deployment, &enterpriseApi.SearchHeadCluster{}, deployment.GetName()+"-shc")).To(Succeed(), "Unable to delete SHC instance") + + // Delete the Indexer Cluster + Expect(testenv.GetAndDeleteCR(ctx, deployment, &enterpriseApi.IndexerCluster{}, deployment.GetName()+"-idxc")).To(Succeed(), "Unable to delete IDXC instance") + + // Delete the Cluster Manager (v3 or v4) + Expect(config.DeleteClusterManager(ctx, deployment)).To(Succeed(), "Unable to delete Cluster Manager") + + // Delete Monitoring Console + Expect(testenv.GetAndDeleteCR(ctx, deployment, mc, mcRef)).To(Succeed(), "Unable to delete Monitoring Console instance") + + Expect(testenv.VerifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, false, verificationTimeout)).To(Succeed()) + + // Verify Monitoring Console PVCs (etc and var) have been deleted + Expect(testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout)).To(Succeed()) +} + +// RunSHCDeployerResourceSpecTest deploys a Search Head Cluster, verifies default CPU limits, +// updates the deployer resource spec, and verifies the deployer is reconfigured while search heads retain defaults. +func RunSHCDeployerResourceSpecTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string) { + shcName := fmt.Sprintf("%s-shc", deployment.GetName()) + _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") + Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "shc", shcName) + + // Verify CPU limits on Search Heads and deployer before updating CR + searchHeadCount := 3 + Expect(testcaseEnvInst.VerifySearchHeadCPULimits(deployment, searchHeadCount, defaultCPULimits)).To(Succeed()) + + deployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) + Expect(testcaseEnvInst.VerifyCPULimits(deployment, deployerPodName, defaultCPULimits)).To(Succeed()) + + shc := &enterpriseApi.SearchHeadCluster{} + Expect(testenv.GetInstanceWithExpect(ctx, deployment, shc, shcName, "Unable to fetch Search Head Cluster deployment")).To(Succeed()) + + // Assign new resources for deployer pod only + newCPULimits := "4" + newCPURequests := "2" + newMemoryLimits := "14Gi" + newMemoryRequests := "12Gi" + + depResSpec := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse(newCPURequests), + "memory": resource.MustParse(newMemoryRequests), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + "memory": resource.MustParse(newMemoryLimits), + }, + } + shc.Spec.DeployerResourceSpec = depResSpec + + Expect(testenv.UpdateCRWithExpect(ctx, deployment, shc, "Unable to deploy Search Head Cluster with updated CR")).To(Succeed()) + + // Verify Search Head go to ready state + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed()) + + // Verify CPU limits on Search Heads - Should be same as before + Expect(testcaseEnvInst.VerifySearchHeadCPULimits(deployment, searchHeadCount, defaultCPULimits)).To(Succeed()) + + // Verify modified deployer spec + Expect(testcaseEnvInst.VerifyResourceConstraints(deployment, deployerPodName, depResSpec)).To(Succeed()) +} + +// RunM4CPUUpdateTest runs the standard M4 CPU limit update test workflow +func RunM4CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Multisite Cluster and Search Head Clusters + mcRef := deployment.GetName() + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + siteCount := 3 + Expect(config.DeployAndVerifyM4(ctx, deployment, testcaseEnvInst, 1, siteCount, mcRef)).To(Succeed(), "Unable to deploy M4 cluster") + + Expect(testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime)).To(Succeed()) + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Verify RF SF is met + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed()) + + // Verify CPU limits on Indexers before updating the CR + Expect(testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, siteCount, defaultCPULimits)).To(Succeed()) + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + for i := 1; i <= siteCount; i++ { + siteName := fmt.Sprintf("site%d", i) + instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) + Expect(testenv.GetInstanceWithExpect(ctx, deployment, idxc, instanceName, "Unable to fetch Indexer Cluster deployment")).To(Succeed()) + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + Expect(testenv.UpdateCRWithExpect(ctx, deployment, idxc, "Unable to deploy Indexer Cluster with updated CR")).To(Succeed()) + } + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-site1" + Expect(testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName)).To(Succeed()) + + // Verify Indexers go to ready state + Expect(testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount)).To(Succeed()) + + // Verify Monitoring Console is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed()) + + // Verify RF SF is met + Expect(testcaseEnvInst.VerifyRFSFMet(ctx, deployment)).To(Succeed()) + + // Verify CPU limits after updating the CR + Expect(testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, siteCount, newCPULimits)).To(Succeed()) +} diff --git a/test/custom_resource_crud/custom_resource_crud_c3_test.go b/test/custom_resource_crud/custom_resource_crud_c3_test.go deleted file mode 100644 index 8c47e63b2..000000000 --- a/test/custom_resource_crud/custom_resource_crud_c3_test.go +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Master - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_m4_test.go b/test/custom_resource_crud/custom_resource_crud_m4_test.go deleted file mode 100644 index 4c00a07dc..000000000 --- a/test/custom_resource_crud/custom_resource_crud_m4_test.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_s1_test.go b/test/custom_resource_crud/custom_resource_crud_s1_test.go deleted file mode 100644 index deb487628..000000000 --- a/test/custom_resource_crud/custom_resource_crud_s1_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1)", func() { - It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", func() { - - // Deploy Standalone - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits before updating the CR - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits) - - // Change CPU limits to trigger CR update - standalone.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") - - // Verify Standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits after updating the CR - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_suite_test.go b/test/custom_resource_crud/custom_resource_crud_suite_test.go index 8972ac76e..d5c657d90 100644 --- a/test/custom_resource_crud/custom_resource_crud_suite_test.go +++ b/test/custom_resource_crud/custom_resource_crud_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,12 +24,12 @@ import ( ) const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond + // DefaultCPULimits is the default CPU limit + DefaultCPULimits = "4" + // UpdatedCPULimits is the updated CPU limit + UpdatedCPULimits = "2" + // DefaultVerificationTimeout is the default timeout for CRUD verification steps + DefaultVerificationTimeout = 150 * time.Second ) var ( @@ -39,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -48,11 +47,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/custom_resource_crud/custom_resource_crud_test.go b/test/custom_resource_crud/custom_resource_crud_test.go new file mode 100644 index 000000000..41303fdc3 --- /dev/null +++ b/test/custom_resource_crud/custom_resource_crud_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// masterManagerCrudConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 and M4 CRUD test tables. +var masterManagerCrudConfigs = []testenv.MasterManagerTestConfig{ + {NamePrefix: "master", Label: "mastercrcrud", NewConfig: testenv.NewClusterReadinessConfigV3}, + {NamePrefix: "", Label: "managercrcrud", NewConfig: testenv.NewClusterReadinessConfigV4}, +} + +var _ = Describe("Custom Resource CRUD test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var defaultCPULimits string + var newCPULimits string + var verificationTimeout time.Duration + + ctx := context.TODO() + + // S1 test — single variant (manager, V4) + Context("Standalone deployment (S1)", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", func() { + RunS1CPUUpdateTest(ctx, deployment, testcaseEnvInst, defaultCPULimits, newCPULimits) + }) + }) + + // C3 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerCrudConfigs { + tc := tc + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + verificationTimeout = DefaultVerificationTimeout + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, c3: can deploy Indexer and Search Head Cluster, change their CR, update the instances", func() { + config := tc.NewConfig() + RunC3CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + + It(tc.Label+", integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { + config := tc.NewConfig() + RunC3PVCDeletionTest(ctx, deployment, testcaseEnvInst, config, verificationTimeout) + }) + }) + } + + // CSPL-3256 - SHC deployer resource spec test (IDXC is irrelevant for this test case) + Context("Search Head Cluster", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { + RunSHCDeployerResourceSpecTest(ctx, deployment, testcaseEnvInst, defaultCPULimits) + }) + }) + + // M4 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerCrudConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, m4: can deploy multisite Indexer and Search Head Clusters, change their CR, update the instances", func() { + config := tc.NewConfig() + RunM4CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + }) + } +}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go b/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go deleted file mode 100644 index 86bcea539..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Search Head Cluster", func() { - // CSPL-3256 - Adding the SHC only test case under c3 as IDXC is irrelevant for this test case - It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { - shcName := fmt.Sprintf("%s-shc", deployment.GetName()) - _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") - if err != nil { - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "Shc", shcName) - } - - // Verify CPU limits on Search Heads and deployer before updating CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - DeployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) - testcaseEnvInst.VerifyCPULimits(deployment, DeployerPodName, defaultCPULimits) - - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - // Assign new resources for deployer pod only - newCPULimits = "4" - newCPURequests := "2" - newMemoryLimits := "14Gi" - newMemoryRequests := "12Gi" - - depResSpec := corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": resource.MustParse(newCPURequests), - "memory": resource.MustParse(newMemoryRequests), - }, - Limits: corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - "memory": resource.MustParse(newMemoryLimits), - }, - } - shc.Spec.DeployerResourceSpec = depResSpec - - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify CPU limits on Search Heads - Should be same as before - searchHeadCount = 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Verify modified deployer spec - testcaseEnvInst.VerifyResourceConstraints(deployment, DeployerPodName, depResSpec) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Manager - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go b/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go deleted file mode 100644 index 89fadcbb8..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/delete_cr/deletecr_suite_test.go b/test/delete_cr/deletecr_suite_test.go index 49b34af39..b997a391b 100644 --- a/test/delete_cr/deletecr_suite_test.go +++ b/test/delete_cr/deletecr_suite_test.go @@ -1,23 +1,20 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "deletecr-" + testenv.RandomDNSName(3) @@ -41,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -50,11 +37,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/delete_cr/deletecr_test.go b/test/delete_cr/deletecr_test.go index 2f3667ba1..34711f14f 100644 --- a/test/delete_cr/deletecr_test.go +++ b/test/delete_cr/deletecr_test.go @@ -1,34 +1,28 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" + "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("DeleteCR test", func() { +var _ = Describe("Delete CR test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment @@ -36,103 +30,23 @@ var _ = Describe("DeleteCR test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) - Context("Standalone deployment (S1 - Standalone Pod)", func() { + Context("Standalone deployment (S1)", func() { It("integration, managerdeletecr: can deploy standalone and delete", func() { - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Delete Standalone CR - err = deployment.DeleteCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to Delete Standalone") - + testcaseEnvInst.RunDeleteStandaloneWorkflow(ctx, deployment) }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3)", func() { It("integration, managerdeletecr: can deploy C3 and delete search head, clustermanager", func() { - - // Deploy C3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, true, "") - Expect(err).To(Succeed(), "Unable to deploy C3 instance") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - idxc := &enterpriseApi.IndexerCluster{} - idxcName := deployment.GetName() + "-idxc" - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Unable to get Indexer instance") - - // Delete Indexer Cluster CR - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to Delete Indexer Cluster") - - sh := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, sh) - Expect(err).To(Succeed(), "Unable to get Search Head instance") - - // Delete Search Head Cluster CR - err = deployment.DeleteCR(ctx, sh) - Expect(err).To(Succeed(), "Unable to Delete Search Head Cluster") - - cm := &enterpriseApi.ClusterManager{} - cmName := deployment.GetName() - err = deployment.GetInstance(ctx, cmName, cm) - Expect(err).To(Succeed(), "Unable to get Cluster Manager instance") - - // Delete Cluster Manager CR - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to Delete Cluster Manager") - + testcaseEnvInst.RunDeleteC3Workflow(ctx, deployment, 3) }) }) }) diff --git a/test/example/example1_test.go b/test/example/example1_test.go index bba0023d1..b2dd4be29 100644 --- a/test/example/example1_test.go +++ b/test/example/example1_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +14,6 @@ package example import ( - "fmt" "math/rand" "time" @@ -32,19 +31,12 @@ var _ = XDescribe("Example1", func() { // This is invoke for each "It" spec below BeforeEach(func() { var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) // "It" spec diff --git a/test/example/example2_test.go b/test/example/example2_test.go index 3988e0976..b68dceaa2 100644 --- a/test/example/example2_test.go +++ b/test/example/example2_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +14,6 @@ package example import ( - "fmt" "math/rand" "time" @@ -32,19 +31,12 @@ var _ = XDescribe("Example2", func() { // This is invoke for each "It" spec below BeforeEach(func() { var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) // "It" spec diff --git a/test/example/example_suite_test.go b/test/example/example_suite_test.go index 78f198f31..bc02a7161 100644 --- a/test/example/example_suite_test.go +++ b/test/example/example_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,9 +14,7 @@ package example import ( - "math/rand" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -29,12 +27,7 @@ var ( testSuiteName = "example-" + testenv.RandomDNSName(3) ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - func TestExampleSuite(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -43,9 +36,11 @@ func TestExampleSuite(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + if testenvInstance != nil { + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") + } }) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go index d2c4be9f1..15b56cf23 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go @@ -11,13 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,18 +25,9 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv - testSuiteName = "indingsep-" + testenv.RandomDNSName(3) + testSuiteName = "idxingsep-" + testenv.RandomDNSName(3) queue = enterpriseApi.QueueSpec{ Provider: "sqs", @@ -100,7 +90,6 @@ var ( appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) s3TestDir = "icappfw-" + testenv.RandomDNSName(4) appListV1 = testenv.BasicApps - s3AppDirV1 = testenv.AppLocationV1 ) // TestBasic is the main entry point @@ -113,19 +102,19 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") appListV1 = testenv.BasicApps appFileList := testenv.GetAppFileList(appListV1) // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download V1 app files") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } err := os.RemoveAll(downloadDirV1) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go index 9142c9882..c2e5dba8c 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "context" @@ -21,38 +21,25 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/onsi/ginkgo/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("indingsep test", func() { +var _ = Describe("Index and Ingestion Separation test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var cmSpec enterpriseApi.ClusterManagerSpec - ctx := context.TODO() BeforeEach(func() { var err error - - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") cmSpec = enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -65,101 +52,22 @@ var _ = Describe("indingsep test", func() { }) AfterEach(func() { - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers", func() { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.SetupIngestorStack(ctx, deployment, queue, objectStorage, cmSpec)).To(Succeed(), "Unable to setup ingestor stack") - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to get Indexer Cluster instance", "Indexer Cluster Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete Indexer Cluster instance", "Indexer Cluster Name", idxc) - - // Delete the Ingestor Cluster - ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Unable to get Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - err = deployment.DeleteCR(ctx, ingest) - Expect(err).To(Succeed(), "Unable to delete Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - - // Delete the Queue - q = &enterpriseApi.Queue{} - err = deployment.GetInstance(ctx, "queue", q) - Expect(err).To(Succeed(), "Unable to get Queue instance", "Queue Name", q) - err = deployment.DeleteCR(ctx, q) - Expect(err).To(Succeed(), "Unable to delete Queue", "Queue Name", q) - - // Delete the ObjectStorage - objStorage = &enterpriseApi.ObjectStorage{} - err = deployment.GetInstance(ctx, "os", objStorage) - Expect(err).To(Succeed(), "Unable to get ObjectStorage instance", "ObjectStorage Name", objStorage) - err = deployment.DeleteCR(ctx, objStorage) - Expect(err).To(Succeed(), "Unable to delete ObjectStorage", "ObjectStorage Name", objStorage) + Expect(testenv.DeleteIngestorStack(ctx, deployment)).To(Succeed(), "Unable to delete ingestor stack") }) - }) - Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers with additional configurations", func() { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers with additional configurations", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") @@ -172,19 +80,13 @@ var _ = Describe("indingsep test", func() { )} queue.SQS.VolList = volumeSpec - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") + // Deploy Queue and ObjectStorage + q, objStorage, err := testenv.DeployQueueAndObjectStorage(ctx, deployment, queue, objectStorage) + Expect(err).To(Succeed(), "Unable to deploy Queue and ObjectStorage") // Deploy Ingestor Cluster with additional configurations (similar to standalone app framework test) appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testcaseEnvInst.GenerateAppFrameworkSpec(ctx, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(5) ic := &enterpriseApi.IngestorCluster{ ObjectMeta: metav1.ObjectMeta{ @@ -257,76 +159,25 @@ var _ = Describe("indingsep test", func() { } allAppSourceInfo := []testenv.AppSourceInfo{ingestorAppSourceInfo} splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + _, err = testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") + Expect(err).To(Succeed(), "Failed to verify app framework state") // Verify probe configuration - testcaseEnvInst.Log.Info("Get config map for probes") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for probes", "ConfigMap", ConfigMapName) - testcaseEnvInst.Log.Info("Verify probe configurations on Ingestor pods") - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + Expect(testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, true)).To(Succeed(), "Probe config verification failed") }) - }) - Context("Ingestor and Indexer deployment", func() { - It("indingsep, integration, indingsep: Splunk Operator can deploy Ingestors and Indexers with correct setup", func() { + It("indexingestionsep, integration: Splunk Operator can deploy Ingestors and Indexers with correct setup", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + Expect(testcaseEnvInst.SetupIngestorStack(ctx, deployment, queue, objectStorage, cmSpec)).To(Succeed(), "Unable to setup ingestor stack") // Get instance of current Ingestor Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Ingestor Cluster CR with latest config") ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Failed to get instance of Ingestor Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, ingest, deployment.GetName()+"-ingest", "Failed to get instance of Ingestor Cluster") // Verify Ingestor Cluster Status testcaseEnvInst.Log.Info("Verify Ingestor Cluster Status") @@ -336,8 +187,7 @@ var _ = Describe("indingsep test", func() { // Get instance of current Indexer Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Indexer Cluster CR with latest config") index := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", index) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, index, deployment.GetName()+"-idxc", "Failed to get instance of Indexer Cluster") // Verify Indexer Cluster Status testcaseEnvInst.Log.Info("Verify Indexer Cluster Status") @@ -352,18 +202,10 @@ var _ = Describe("indingsep test", func() { if strings.Contains(pod, "ingest") || strings.Contains(pod, "idxc") { // Verify outputs.conf - testcaseEnvInst.Log.Info("Verify outputs.conf") - outputsPath := "opt/splunk/etc/system/local/outputs.conf" - outputsConf, err := testenv.GetConfFile(pod, outputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get outputs.conf from Ingestor Cluster pod") - testenv.ValidateContent(outputsConf, outputs, true) + testenv.VerifyConfFileContent(pod, "opt/splunk/etc/system/local/outputs.conf", deployment.GetName(), outputs, "Failed to get outputs.conf from Ingestor Cluster pod") // Verify default-mode.conf - testcaseEnvInst.Log.Info("Verify default-mode.conf") - defaultsPath := "opt/splunk/etc/system/local/default-mode.conf" - defaultsConf, err := testenv.GetConfFile(pod, defaultsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get default-mode.conf from Ingestor Cluster pod") - testenv.ValidateContent(defaultsConf, defaultsAll, true) + testenv.VerifyConfFileContent(pod, "opt/splunk/etc/system/local/default-mode.conf", deployment.GetName(), defaultsAll, "Failed to get default-mode.conf from Ingestor Cluster pod") // Verify AWS env variables testcaseEnvInst.Log.Info("Verify AWS env variables") @@ -378,11 +220,7 @@ var _ = Describe("indingsep test", func() { testenv.ValidateContent(defaultsConf, defaultsIngest, true) } else if strings.Contains(pod, "idxc") { // Verify inputs.conf - testcaseEnvInst.Log.Info("Verify inputs.conf") - inputsPath := "opt/splunk/etc/system/local/inputs.conf" - inputsConf, err := testenv.GetConfFile(pod, inputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get inputs.conf from Indexer Cluster pod") - testenv.ValidateContent(inputsConf, inputs, true) + testenv.VerifyConfFileContent(pod, "opt/splunk/etc/system/local/inputs.conf", deployment.GetName(), inputs, "Failed to get inputs.conf from Indexer Cluster pod") } } }) diff --git a/test/ingest_search/ingest_search_suite_test.go b/test/ingest_search/ingest_search_suite_test.go index 301f11611..965b0cd56 100644 --- a/test/ingest_search/ingest_search_suite_test.go +++ b/test/ingest_search/ingest_search_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package ingestsearchtest import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "ingest-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -48,11 +37,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/ingest_search/ingest_search_test.go b/test/ingest_search/ingest_search_test.go index 090f034a9..e8cc3b53d 100644 --- a/test/ingest_search/ingest_search_test.go +++ b/test/ingest_search/ingest_search_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,253 +14,37 @@ package ingestsearchtest import ( - "bufio" "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("Ingest and Search Test", func() { +var _ = Describe("Ingest and Search test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var firstLine string ctx := context.TODO() BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can search internal logs for standalone instance", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - searchString := "index=_internal | stats count by host" - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) - - var searchResults map[string]interface{} - unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - if unmarshalErr != nil { - testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") - } - - prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - searchString := "index=_internal GUID component=ServerConfig" - - // Perform a simple search - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString, deployment) - if reqErr != nil { - testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - if statusErr != nil { - testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - if resErr != nil { - testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - - // Display results for debug purposes - prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + RunS1InternalLogSearchTest(ctx, deployment, testcaseEnvInst) }) - }) - Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can ingest custom data to new index and search", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify splunk status is up - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - splunkBin := "/opt/splunk/bin/splunk" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - splunkCmd := "status" - - statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) - command := []string{"/bin/bash"} - statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) - return enterpriseApi.PhaseError - } - - if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { - testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) - return enterpriseApi.PhaseError - } - - testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - // Create an index - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - indexName := "myTestIndex" - - // Create an index on a standalone instance - err = testenv.CreateAnIndexStandalone(ctx, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed response to add index to splunk") - - // Create a mock logfile to ingest - logFile := "/tmp/test.log" - err = testenv.CreateMockLogfile(logFile, 1) - Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) - - // Copy log file and ingest it - err = testenv.IngestFileViaOneshot(ctx, logFile, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) - - // Read first line to find a search token - var file, openErr = os.Open(logFile) - Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) - - reader := bufio.NewReader(file) - var readErr error - firstLine, readErr = reader.ReadString('\n') - Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) - - tokens := strings.Fields(firstLine) - Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) - - searchToken := tokens[len(tokens)-1] - testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) - - searchString := fmt.Sprintf("index=%s | stats count by host", indexName) - - // Wait for search results to be available instead of fixed sleep - err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for search results") - - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) - - // Verify result. Should get count 1. result:{count:1} - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) - - hostCount := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) - testHostCnt := strings.Compare(hostCount["count"].(string), "1") - testHostname := strings.Compare(hostCount["host"].(string), podName) - Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) - - searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString2, deployment) - Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) - - testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) - var searchResults2 testenv.SearchJobResultsResponse - jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - found := false - for key, elem := range searchResults2.Results { - testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) - trimFirstLine := strings.TrimSuffix(firstLine, "\n") - if strings.Compare(elem.Raw, trimFirstLine) == 0 { - testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) - found = true - } - } - Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) + RunS1IngestAndSearchTest(ctx, deployment, testcaseEnvInst) }) }) }) diff --git a/test/ingest_search/ingest_search_test_shared.go b/test/ingest_search/ingest_search_test_shared.go new file mode 100644 index 000000000..20ce6a2ce --- /dev/null +++ b/test/ingest_search/ingest_search_test_shared.go @@ -0,0 +1,206 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ingestsearchtest + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// RunS1InternalLogSearchTest deploys a Standalone instance and verifies internal log searches +// using both synchronous and asynchronous search APIs. +func RunS1InternalLogSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + standalone, err := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, "", "") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + + searchString := "index=_internal | stats count by host" + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, podName, searchString) + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) + + var searchResults map[string]interface{} + unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + if unmarshalErr != nil { + testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") + } + + prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") + if jsonErr != nil { + testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") + } else { + testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) + } + + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + searchString := "index=_internal GUID component=ServerConfig" + + sid, reqErr := testenv.PerformSearchReq(ctx, deployment, podName, searchString) + if reqErr != nil { + testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, deployment, podName, sid) + if statusErr != nil { + testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, deployment, podName, sid) + if resErr != nil { + testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) + return enterpriseApi.PhaseError + } + + prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") + if jsonErr != nil { + testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") + } else { + testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) + } + + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) +} + +// RunS1IngestAndSearchTest deploys a Standalone instance, ingests a custom log file into a new +// index, and verifies the ingested data is searchable via both sync and async search APIs. +func RunS1IngestAndSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + standalone, err := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, "", "") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + + splunkBin := "/opt/splunk/bin/splunk" + username := "admin" + password := "$(cat /mnt/splunk-secrets/password)" + splunkCmd := "status" + + statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) + command := []string{"/bin/bash"} + statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) + return enterpriseApi.PhaseError + } + + if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { + testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) + return enterpriseApi.PhaseError + } + + testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "phase", standalone.Status.Phase) + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + indexName := "myTestIndex" + + err = testenv.CreateAnIndexStandalone(ctx, deployment, indexName, podName) + Expect(err).To(Succeed(), "Failed response to add index to splunk") + + logFile := "/tmp/test.log" + err = testenv.CreateMockLogfile(logFile, 1) + Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) + + err = testenv.IngestFileViaOneshot(ctx, deployment, logFile, indexName, podName) + Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) + + file, openErr := os.Open(logFile) + Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) + + reader := bufio.NewReader(file) + firstLine, readErr := reader.ReadString('\n') + Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) + + tokens := strings.Fields(firstLine) + Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) + + searchToken := tokens[len(tokens)-1] + testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) + + searchString := fmt.Sprintf("index=%s | stats count by host", indexName) + + err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) + Expect(err).To(Succeed(), "Timed out waiting for search results") + + searchResultsResp, err := testenv.PerformSearchSync(ctx, deployment, podName, searchString) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) + + var searchResults map[string]interface{} + jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) + + hostCount := searchResults["result"].(map[string]interface{}) + testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) + testHostCnt := strings.Compare(hostCount["count"].(string), "1") + testHostname := strings.Compare(hostCount["host"].(string), podName) + Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) + Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) + + searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) + sid, reqErr := testenv.PerformSearchReq(ctx, deployment, podName, searchString2) + Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) + testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, deployment, podName, sid) + Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) + testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, deployment, podName, sid) + Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) + + testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) + var searchResults2 testenv.SearchJobResultsResponse + jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + found := false + for key, elem := range searchResults2.Results { + testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) + trimFirstLine := strings.TrimSuffix(firstLine, "\n") + if strings.Compare(elem.Raw, trimFirstLine) == 0 { + testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) + found = true + } + } + Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) +} diff --git a/test/licensemanager/manager_suite_test.go b/test/licensemanager/licensemanager_suite_test.go similarity index 71% rename from test/licensemanager/manager_suite_test.go rename to test/licensemanager/licensemanager_suite_test.go index d504c3685..35831873a 100644 --- a/test/licensemanager/manager_suite_test.go +++ b/test/licensemanager/licensemanager_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package licensemanager import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "lmanager-" + testenv.RandomDNSName(3) @@ -47,11 +37,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/licensemanager/lm_appfw_test.go b/test/licensemanager/lm_appfw_test.go new file mode 100644 index 000000000..f8dba0168 --- /dev/null +++ b/test/licensemanager/lm_appfw_test.go @@ -0,0 +1,50 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package licensemanager + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("License Manager App Framework test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + for _, tc := range masterManagerLMConfigs { + tc := tc + + Context("Clustered deployment (C3) with "+tc.Label+" App Framework", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { + RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, tc.NewConfig()) + }) + }) + } +}) diff --git a/test/licensemanager/lm_s1_test.go b/test/licensemanager/lm_s1_test.go deleted file mode 100644 index 035ab730f..000000000 --- a/test/licensemanager/lm_s1_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with License Manager", func() { - It("licensemanager, smoke, s1: Splunk Operator can configure License Manager with Standalone in S1 SVA", func() { - RunLMS1Test(ctx, deployment, testcaseEnvInst, config) - }) - }) -}) diff --git a/test/licensemanager/lm_test.go b/test/licensemanager/lm_test.go new file mode 100644 index 000000000..133c6cb0a --- /dev/null +++ b/test/licensemanager/lm_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package licensemanager + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// masterManagerLMConfigs defines the V3 (master) and V4 (manager) variants +// for the license manager tests. +var masterManagerLMConfigs = []struct { + NamePrefix string + Label string + NewConfig func() *testenv.LicenseTestConfig +}{ + {"master", "licensemaster", testenv.NewLicenseMasterConfig}, + {"", "licensemanager", testenv.NewLicenseManagerConfig}, +} + +var _ = Describe("License Manager test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + for _, tc := range masterManagerLMConfigs { + tc := tc + + Context("Standalone deployment (S1) with "+tc.Label, func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", smoke, s1: Splunk Operator can configure LM with Standalone in S1 SVA", func() { + RunLMS1Test(ctx, deployment, testcaseEnvInst, tc.NewConfig()) + }) + }) + + Context("Clustered deployment (C3) with "+tc.Label, func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, c3: Splunk Operator can configure LM with Indexers and Search Heads in C3 SVA", func() { + RunLMC3Test(ctx, deployment, testcaseEnvInst, tc.NewConfig()) + }) + }) + + Context("Multisite cluster deployment (M4) with "+tc.Label, func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, m4: Splunk Operator can configure LM with Indexers and Search Heads in M4 SVA", func() { + RunLMM4Test(ctx, deployment, testcaseEnvInst, tc.NewConfig()) + }) + }) + } +}) diff --git a/test/licensemanager/lm_test_shared.go b/test/licensemanager/lm_test_shared.go index 205fd2c74..a7a65754b 100644 --- a/test/licensemanager/lm_test_shared.go +++ b/test/licensemanager/lm_test_shared.go @@ -20,238 +20,67 @@ import ( "path/filepath" "time" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + . "github.com/onsi/gomega" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" - - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" ) -type LicenseTestConfig struct { - DeployStandaloneWithLM func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) - LicenseManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - ClusterManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - DeployMultisiteClusterWithSearchHead func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas, siteCount int, mcRef string) error - DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) - LicenseManagerPodName string - LicenseManagerSpecType string -} - -func NewLicenseMasterConfig() *LicenseTestConfig { - return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterMasterReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, - DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { - return deployment.DeployLicenseMasterWithGivenSpec(ctx, name, spec.(enterpriseApiV3.LicenseMasterSpec)) - }, - LicenseManagerPodName: testenv.LicenseMasterPod, - LicenseManagerSpecType: "v3", - } -} - -func NewLicenseManagerConfig() *LicenseTestConfig { - return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLM(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterManagerReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, - DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { - return deployment.DeployLicenseManagerWithGivenSpec(ctx, name, spec.(enterpriseApi.LicenseManagerSpec)) - }, - LicenseManagerPodName: testenv.LicenseManagerPod, - LicenseManagerSpecType: "v4", - } -} - -func downloadLicenseAndCreateConfigMap(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv) { - downloadDir := "licenseFolder" - - var err error - var licenseFilePath string - - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err = testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - case "azure": - licenseFilePath, err = testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - case "gcp": - licenseFilePath, err = testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) -} - -func downloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) { - switch testenv.ClusterProvider { - case "eks": - err := testenv.DownloadFilesFromS3(testDataS3Bucket, appDir, downloadDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) - case "azure": - containerName := "/" + azureDataContainer + "/" + appDir - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDir, containerName, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) - case "gcp": - err := testenv.DownloadFilesFromGCP(testDataS3Bucket, appDir, downloadDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) - } -} - -func uploadAppFiles(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, testS3Bucket, testDir, downloadDir string, appFileList []string, version string) []string { - var uploadedFiles []string - var err error - - switch testenv.ClusterProvider { - case "eks": - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", version)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, testDir, appFileList, downloadDir) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3", version)) - case "azure": - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", version)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDir, testDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure", version)) - case "gcp": - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP", version)) - uploadedFiles, err = testenv.UploadFilesToGCP(testS3Bucket, testDir, appFileList, downloadDir) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP", version)) - } - - return uploadedFiles -} - -func deleteUploadedFiles(ctx context.Context, testS3Bucket string, uploadedApps []string) { - switch testenv.ClusterProvider { - case "eks": - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - case "azure": - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - case "gcp": - testenv.DeleteFilesOnGCP(testS3Bucket, uploadedApps) - } -} - -func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) +// RunLMS1Test deploys a Standalone with License Manager and Monitoring Console, +// then verifies LM is configured on the standalone and MC pods. +func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.LicenseTestConfig) { + // Set up license config map + Expect(testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst)).To(Succeed(), "Unable to setup license config map") - // Create standalone Deployment with License Manager/Master + // Create Standalone deployment with License Manager/Master mcRef := deployment.GetName() standalone, err := config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with LM") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + Expect(config.LicenseManagerReady(ctx, deployment, testcaseEnvInst)).To(Succeed()) // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed()) - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) + // Deploy and verify Monitoring Console + _, err = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Verify livenessProbe and readinessProbe config object and scripts + Expect(testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, false)).To(Succeed(), "Probe config verification failed") - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify License Manager/Master is configured on standalone instance + // Verify License Manager/Master is configured on Standalone instance standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, standalonePodName) + Expect(testenv.VerifyLMConfiguredOnPod(ctx, deployment, standalonePodName)).To(Succeed()) // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + Expect(testenv.VerifyLMConfiguredOnMC(ctx, deployment)).To(Succeed()) } -func RunLMC3Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) - - // Deploy Single site Cluster with License Manager/Master +// RunLMC3Test deploys a C3 cluster with License Manager and Monitoring Console, +// then verifies LM is configured on indexers, search heads, and MC. +func RunLMC3Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.LicenseTestConfig) { + // Deploy single site Cluster with License Manager/Master mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + Expect(config.DeployC3WithLicense(ctx, deployment, testcaseEnvInst, 3, true, mcRef)).To(Succeed(), "Unable to deploy C3 with license") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + Expect(testenv.DeployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef)).To(Succeed(), "Unable to deploy MC and verify RFSF") - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + // Verify License Manager/Master is configured on indexers, search heads, and MC + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), 3, false, 0) + Expect(testenv.VerifyLMConfiguredOnCluster(ctx, deployment, indexerPods)).To(Succeed()) } -func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, testenvInstance *testenv.TestEnv, config *LicenseTestConfig) { +// RunLMC3AppFrameworkTest deploys a License Manager with App Framework, verifies V1 apps +// are installed, upgrades to V2 apps, and verifies the updated apps. +func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, testenvInstance *testenv.TestEnv, config *testenv.LicenseTestConfig) { var ( appListV1 []string appListV2 []string testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") testDataS3Bucket = os.Getenv("TEST_BUCKET") - AzureDataContainer = os.Getenv("TEST_CONTAINER") + azureDataContainer = os.Getenv("TEST_CONTAINER") appDirV1 = testenv.AppLocationV1 appDirV2 = testenv.AppLocationV2 currDir, _ = os.Getwd() @@ -267,39 +96,32 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment appFileList := testenv.GetAppFileList(appListV1) // Download V1 Apps - downloadAppFiles(ctx, testDataS3Bucket, AzureDataContainer, appDirV1, downloadDirV1, appFileList, appVersion) + Expect(testenv.DownloadAppFiles(ctx, testDataS3Bucket, azureDataContainer, appDirV1, downloadDirV1, appFileList, appVersion)).To(Succeed(), "Unable to download V1 app files") // Upload V1 apps testDir = "lm-" + testenv.RandomDNSName(4) - uploadedFiles := uploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV1, appFileList, appVersion) + uploadedFiles, err := testenv.UploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV1, appFileList, appVersion) + Expect(err).To(Succeed(), "Unable to upload V1 app files") uploadedApps = append(uploadedApps, uploadedFiles...) - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Set up license config map + Expect(testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst)).To(Succeed(), "Unable to setup license config map") - // Create App framework Spec + // Create app framework spec volumeName := "lm-test-volume-" + testenv.RandomDNSName(3) - var volumeSpec []enterpriseApi.VolumeSpec - switch testenv.ClusterProvider { - case "eks": - volumeSpec = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - case "azure": - volumeSpec = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(volumeName, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - case "gcp": - volumeSpec = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "gcp", "blob", testenv.GetDefaultS3Region())} - } + volumeSpec := testcaseEnvInst.GenerateVolumeSpecForProvider(ctx, volumeName) - // AppSourceDefaultSpec: Remote Storage volume name and Scope of App deployment + // AppSourceDefaultSpec: Remote Storage volume name and scope of app deployment appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: volumeName, Scope: enterpriseApi.ScopeLocal, } - // appSourceSpec: App source name, location and volume name and scope from appSourceDefaultSpec + // appSourceSpec: app source name, location and volume name and scope from appSourceDefaultSpec appSourceName := "lm-" + testenv.RandomDNSName(3) appSourceSpec := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceName, testDir, appSourceDefaultSpec)} - // appFrameworkSpec: AppSource settings, Poll Interval, volumes, appSources on volumes + // appFrameworkSpec: AppSource settings, poll interval, volumes, appSources on volumes appFrameworkSpec := enterpriseApi.AppFrameworkSpec{ Defaults: appSourceDefaultSpec, AppsRepoPollInterval: 60, @@ -307,72 +129,25 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment AppSources: appSourceSpec, } - var spec interface{} - if config.LicenseManagerSpecType == "v3" { - spec = enterpriseApiV3.LicenseMasterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } else { - spec = enterpriseApi.LicenseManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } + spec := config.BuildLMAppFrameworkSpec(testcaseEnvInst, appFrameworkSpec) // Deploy the License Manager/Master with App Framework - var err error _, err = config.DeployLicenseManagerWithGivenSpec(ctx, deployment, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy LM with App framework") + Expect(err).To(Succeed(), "Unable to deploy LM with App Framework") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + Expect(config.LicenseManagerReady(ctx, deployment, testcaseEnvInst)).To(Succeed()) - // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) - podName := []string{fmt.Sprintf(config.LicenseManagerPodName, deployment.GetName(), 0)} - testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appListV1, true, enterpriseApi.ScopeLocal) + // Wait for V1 apps to reach Install phase on License Manager/Master + err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, deployment.GetName(), config.CrKind, appSourceName, appListV1, enterpriseApi.PhaseInstall, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for V1 apps to reach Install phase on LicenseManager") - // Verify apps are installed on License Manager/Master - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appListV1, false, "enabled", false, false) + // Verify apps are copied and installed on License Manager/Master + podName := []string{fmt.Sprintf(config.LicenseManagerPodName, deployment.GetName(), 0)} + Expect(testenv.VerifyLMAppsOnPod(ctx, deployment, testcaseEnvInst, testenvInstance, podName, appListV1, false)).To(Succeed()) // Delete files uploaded - deleteUploadedFiles(ctx, testS3Bucket, uploadedApps) + testenv.DeleteUploadedFiles(ctx, testS3Bucket, uploadedApps) uploadedApps = nil // Create a list of apps to upload to S3 after poll period @@ -381,81 +156,45 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment appVersion = "V2" // Download V2 Apps - downloadAppFiles(ctx, testDataS3Bucket, AzureDataContainer, appDirV2, downloadDirV2, appFileList, appVersion) + Expect(testenv.DownloadAppFiles(ctx, testDataS3Bucket, azureDataContainer, appDirV2, downloadDirV2, appFileList, appVersion)).To(Succeed(), "Unable to download V2 app files") // Upload V2 apps - uploadedFiles = uploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV2, appFileList, appVersion) + uploadedFiles, err = testenv.UploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV2, appFileList, appVersion) + Expect(err).To(Succeed(), "Unable to upload V2 app files") uploadedApps = append(uploadedApps, uploadedFiles...) - err = testcaseEnvInst.WaitForLicenseManagerPhase(ctx, deployment, testcaseEnvInst.GetName(), deployment.GetName(), enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for LicenseManager to reach Ready phase") + // Wait for operator to detect V2 apps (any app leaves Install phase) + testcaseEnvInst.WaitforPhaseChange(ctx, deployment, deployment.GetName(), config.CrKind, appSourceName, appFileList) - // Verify LM stays in ready state - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + // Wait for License Manager/Master to finish processing V2 apps and become Ready + Expect(config.LicenseManagerReady(ctx, deployment, testcaseEnvInst)).To(Succeed()) - // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) - testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appListV2, true, enterpriseApi.ScopeLocal) + // Wait for V2 apps to reach Install phase on License Manager/Master + err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, deployment.GetName(), config.CrKind, appSourceName, appListV2, enterpriseApi.PhaseInstall, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for V2 apps to reach Install phase on LicenseManager") - // Verify apps are installed on License Manager/Master - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appListV2, true, "enabled", true, false) + // Verify apps are copied and installed on License Manager/Master + Expect(testenv.VerifyLMAppsOnPod(ctx, deployment, testcaseEnvInst, testenvInstance, podName, appListV2, true)).To(Succeed()) // Delete files uploaded - deleteUploadedFiles(ctx, testS3Bucket, uploadedApps) + testenv.DeleteUploadedFiles(ctx, testS3Bucket, uploadedApps) // Delete locally downloaded app files os.RemoveAll(downloadDirV1) os.RemoveAll(downloadDirV2) } -func RunLMM4Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) - +// RunLMM4Test deploys a multisite cluster with License Manager and Monitoring Console, +// then verifies LM is configured on indexers, search heads, and MC. +func RunLMM4Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.LicenseTestConfig) { // Deploy Multisite Cluster with License Manager/Master and Search Head siteCount := 3 mcRef := deployment.GetName() - err := config.DeployMultisiteClusterWithSearchHead(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(config.DeployM4WithLicense(ctx, deployment, testcaseEnvInst, 1, siteCount, mcRef)).To(Succeed(), "Unable to deploy M4 with license") - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(testenv.DeployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef)).To(Succeed(), "Unable to deploy MC and verify RFSF") - // Verify Multisite Indexer Cluster status - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 2, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 3, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + // Verify License Manager/Master is configured on indexers, search heads, and MC + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) + Expect(testenv.VerifyLMConfiguredOnCluster(ctx, deployment, indexerPods)).To(Succeed()) } diff --git a/test/licensemanager/manager_lm_c3_test.go b/test/licensemanager/manager_lm_c3_test.go deleted file mode 100644 index 8e44449c9..000000000 --- a/test/licensemanager/manager_lm_c3_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure License Manager with Indexers and Search Heads in C3 SVA", func() { - RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { - RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemanager/manager_lm_m4_test.go b/test/licensemanager/manager_lm_m4_test.go deleted file mode 100644 index aedcd4dee..000000000 --- a/test/licensemanager/manager_lm_m4_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Manager", func() { - It("licensemanager, integration, m4: Splunk Operator can configure License Manager with indexers and search head in M4 SVA", func() { - RunLMM4Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - -}) diff --git a/test/licensemaster/lm_c3_test.go b/test/licensemaster/lm_c3_test.go deleted file mode 100644 index ed2f83f7f..000000000 --- a/test/licensemaster/lm_c3_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() { - licensemanager.RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { - licensemanager.RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemaster/lm_m4_test.go b/test/licensemaster/lm_m4_test.go deleted file mode 100644 index 1e4c6fb65..000000000 --- a/test/licensemaster/lm_m4_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Master", func() { - It("licensemaster, integration, m4: Splunk Operator can configure License Master with indexers and search head in M4 SVA", func() { - licensemanager.RunLMM4Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - -}) diff --git a/test/licensemaster/lm_s1_test.go b/test/licensemaster/lm_s1_test.go deleted file mode 100644 index cc5c92e6b..000000000 --- a/test/licensemaster/lm_s1_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with License Master", func() { - It("licensemaster, smoke, s1: Splunk Operator can configure License Master with Standalone in S1 SVA", func() { - licensemanager.RunLMS1Test(ctx, deployment, testcaseEnvInst, config) - }) - }) -}) diff --git a/test/licensemaster/lm_suite_test.go b/test/licensemaster/lm_suite_test.go deleted file mode 100644 index 9df3f4576..000000000 --- a/test/licensemaster/lm_suite_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "lm-" + testenv.RandomDNSName(3) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } -}) diff --git a/test/monitoring_console/manager_monitoring_console_test.go b/test/monitoring_console/manager_monitoring_console_test.go deleted file mode 100644 index 1cc3b89f3..000000000 --- a/test/monitoring_console/manager_monitoring_console_test.go +++ /dev/null @@ -1,1101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package monitoringconsoletest - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Monitoring Console test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Deploy Monitoring Console", func() { - It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", func() { - /* - Test Steps - 1. Deploy Monitoring Console - 2. Deploy Standalone - 3. Wait for Monitoring Console status to go back to READY - 4. Verify Standalone configured in Monitoring Console Config Map - 5. Verify Monitoring Console Pod has correct peers in Peer List - --------------- RECONFIG WITH NEW MC -------------------------- - 6. Reconfig S1 with 2nd Monitoring Console Name - 7. Check 2nd Monitoring Console Config Map to verify s1 - 8. Deploy 2nd Monitoring Console Pod - 9. Verify Standalone pod is configured on Monitoring Console Pod - 10. Verify 1st Monitoring Console Config Map is not configured with S1 - 11. Verify 1st Monitoring Console Pod is not configured with S1 - */ - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Create Standalone Spec and apply - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### - - // Reconfig S1 with 2nd Monitoring Console Name - mcTwoName := deployment.GetName() + "-two" - err = deployment.GetInstance(ctx, standaloneOneName, standaloneOne) - Expect(err).To(Succeed(), "Unable to get instance of Standalone") - standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName - - // Update Standalone with 2nd MC - err = deployment.UpdateCR(ctx, standaloneOne) - Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") - - // Deploy 2nd MC Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Second Monitoring Console Pod") - - // Verify 2nd Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on SECOND MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcTwoName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in SECOND MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcTwoName, true, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is not configured in MC ONE Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod NOT ON FIRST MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod Not in MC ONE Peer List - testcaseEnvInst.Log.Info("Check standalone NOT ON FIRST MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", func() { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - // CSPL-901 standaloneTwoName := deployment.GetName() + "-two" - standaloneTwoName := "standalone-" + testenv.RandomDNSName(3) - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete Standlone TWO of the standalone and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment with Scale up", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Scale Standalone to 2 REPLICAS - 8. Wait for Second Standalone POD to come up and PHASE to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone PODS configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 1, false, 0) - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Scale Standalone instance - testcaseEnvInst.Log.Info("Scaling Standalone CR") - scaledReplicaCount := 2 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale Standalone") - - // Ensure standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings - 6. Verify Monitoring Console Pod has peers(indexers) in Peer string - 7. Scale SH Cluster - 8. Scale Indexer Count - 9. Add a standalone - 10. Verify Standalone is configured in MC Config Map and Peer String - 11. Verify SH are configured in MC Config Map and Peers String - 12. Verify Indexers are configured in Peer String - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Cluster Manager to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager in MC ConfigMap") - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // Scale Search Head Cluster - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Scale indexers - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready Phase - // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) - - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc1, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ################# Update Monitoring Console In Cluster Manager CR ################################## - - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Manager Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // ################# Update Monitoring Console In SHC CR ################################## - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") - - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify MC is Ready and stays in ready state - // testenv.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo, testcaseEnvInst) - - // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) - - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC two config after SHC reconfig") - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for indexers in MC two config after SHC reconfig") - - // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) - - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads to be removed from MC one config after SHC reconfig") - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - }) - }) - - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("managermc2, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { - /* - Test Steps - 1. Deploy Multisite Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Manager to use 2nd Monitoring Console - 9. Verify Cluster Manager is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Manager not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod - */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Manager RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", func() { - /* - Test Steps - 1. Deploy Standalone with name "search-head-adhoc" - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone with name "search-head" - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := "search-head-adhoc" - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standaloneOne to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneOneName, standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneOne is configured in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneOne Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - standaloneTwoName := "search-head" - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource(CR) resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneTwo is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneTwo Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete standaloneTwo and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - -}) diff --git a/test/monitoring_console/monitoring_console_suite_test.go b/test/monitoring_console/monitoring_console_suite_test.go index 83bf2060d..b9277aa25 100644 --- a/test/monitoring_console/monitoring_console_suite_test.go +++ b/test/monitoring_console/monitoring_console_suite_test.go @@ -1,23 +1,20 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package monitoringconsoletest import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "mc-" + testenv.RandomDNSName(3) @@ -41,23 +29,19 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/monitoring_console/monitoring_console_test.go b/test/monitoring_console/monitoring_console_test.go index 87605c825..d5b39aba6 100644 --- a/test/monitoring_console/monitoring_console_test.go +++ b/test/monitoring_console/monitoring_console_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,20 +15,85 @@ package monitoringconsoletest import ( "context" - "fmt" "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) -var _ = Describe("Monitoring Console test", func() { +// masterManagerMCConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 reconfig and M4 MC reconfig test tables. +var masterManagerMCConfigs = []testenv.MCVersionConfig{ + { + MCReconfigParams: testenv.MCReconfigParams{CMServiceNameFmt: testenv.ClusterMasterServiceName, CMURLKey: "SPLUNK_CLUSTER_MASTER_URL"}, + NamePrefix: "master", + Label: "mastermc", + DeployC3WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, shc bool, mcRef string) error { + return d.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, name, replicas, shc, mcRef) + }, + DeployM4WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error { + return d.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, name, replicas, siteCount, mcRef, shc) + }, + NewCMObject: func() interface{} { return &enterpriseApiV3.ClusterMaster{} }, + VerifyCMReady: func(ctx context.Context, d *testenv.Deployment, te *testenv.TestCaseEnv) error { + return te.VerifyClusterMasterReady(ctx, d) + }, + SHCReconfigTimeout: 0, + VerifyMCTwoReadyAfterSHC: true, + }, + { + MCReconfigParams: testenv.MCReconfigParams{CMServiceNameFmt: testenv.ClusterManagerServiceName, CMURLKey: splcommon.ClusterManagerURL}, + NamePrefix: "", + Label: "managermc", + DeployC3WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, shc bool, mcRef string) error { + return d.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, name, replicas, shc, mcRef) + }, + DeployM4WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error { + return d.DeployMultisiteClusterWithMonitoringConsole(ctx, name, replicas, siteCount, mcRef, shc) + }, + NewCMObject: func() interface{} { return &enterpriseApi.ClusterManager{} }, + VerifyCMReady: func(ctx context.Context, d *testenv.Deployment, te *testenv.TestCaseEnv) error { + return te.VerifyClusterManagerReady(ctx, d) + }, + SHCReconfigTimeout: 5 * time.Minute, + VerifyMCTwoReadyAfterSHC: false, + }, +} + +// C3 scale-up tests — V3 (master) and V4 (manager) variants +var _ = Describe("Monitoring Console C3 scale-up tests", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + for _, cfg := range masterManagerMCConfigs { + cfg := cfg + Context("Clustered deployment C3 scale-up ("+cfg.Label+")", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, cfg.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(cfg.Label+", smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { + RunC3MCScaleUpTest(ctx, deployment, testcaseEnvInst, cfg) + }) + }) + } +}) + +// Manager (V4) Monitoring Console tests +var _ = Describe("Monitoring Console test (manager)", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment @@ -36,551 +101,197 @@ var _ = Describe("Monitoring Console test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { + Context("Deploy Monitoring Console", func() { + It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", func() { /* Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console + 1. Deploy Monitoring Console + 2. Deploy Standalone 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings - 6. Verify Monitoring Console Pod has peers(indexers) in Peer string - 7. Scale SH Cluster - 8. Scale Indexer Count - 9. Add a standalone - 10. Verify Standalone is configured in MC Config Map and Peer String - 11. Verify SH are configured in MC Config Map and Peers String - 12. Verify Indexers are configured in Peer String + 4. Verify Standalone configured in Monitoring Console Config Map + 5. Verify Monitoring Console Pod has correct peers in Peer List + --------------- RECONFIG WITH NEW MC -------------------------- + 6. Reconfig S1 with 2nd Monitoring Console Name + 7. Check 2nd Monitoring Console Config Map to verify s1 + 8. Deploy 2nd Monitoring Console Pod + 9. Verify Standalone pod is configured on Monitoring Console Pod + 10. Verify 1st Monitoring Console Config Map is not configured with S1 + 11. Verify 1st Monitoring Console Pod is not configured with S1 */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Cluster Master to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Master in MC ConfigMap") - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // Scale Search Head Cluster - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Scale indexers - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") + // Deploy Monitoring Console CRD + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + // Create Standalone Spec and apply + standaloneOneName := deployment.GetName() + mcName := deployment.GetName() + standaloneOne, err := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneOneName, mcName) + Expect(err).To(Succeed(), "Unable to deploy Standalone with MC reference") - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // wait for custom resource resource version to change and verify MC is ready + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed()) - // Ensure Search Head Cluster go to Ready Phase - // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Check Standalone is configured in MC + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcName, true)).To(Succeed()) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) + // Reconfig S1 with 2nd Monitoring Console Name + mcTwoName := deployment.GetName() + "-two" + Expect(testenv.GetInstanceWithExpect(ctx, deployment, standaloneOne, standaloneOneName, "Unable to get instance of Standalone")).To(Succeed()) + standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Update Standalone with 2nd MC + err = deployment.UpdateCR(ctx, standaloneOne) + Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) + // Deploy 2nd MC Pod + _, err = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two") - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) + // Check Standalone is configured in MC Two + testcaseEnvInst.Log.Info("Checking for Standalone on SECOND MC after Standalone RECONFIG") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcTwoName, true)).To(Succeed()) - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) + // Verify Monitoring Console One is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed()) - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + // Check Standalone is NOT configured in MC One + testcaseEnvInst.Log.Info("Checking for Standalone NOT ON FIRST MC after Standalone RECONFIG") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcName, false)).To(Succeed()) - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + }) + }) - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + Context("Standalone deployment (S1)", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", func() { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, deployment.GetName(), "standalone-"+testenv.RandomDNSName(3)) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { + Context("Standalone deployment with Scale up", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() { /* Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod + 1. Deploy Standalone + 2. Wait for Standalone to go to READY + 3. Deploy Monitoring Console + 4. Wait for Monitoring Console status to be READY + 5. Verify Standalone configured in Monitoring Console Config Map + 6. Verify Monitoring Console Pod has correct peers in Peer List + 7. Scale Standalone to 2 REPLICAS + 8. Wait for Second Standalone POD to come up and PHASE to be READY + 9. Wait for Monitoring Console status to go UPDATING then READY + 10. Verify both Standalone PODS configured in Monitoring Console Config Map + 11. Verify both Standalone configured in Monitoring Console Pod Peers String */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 + standaloneName := deployment.GetName() mcName := deployment.GetName() + standalone, err := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneName, mcName) + Expect(err).To(Succeed(), "Unable to deploy Standalone with MC reference") - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") + // Deploy MC and wait for MC to be READY + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Check Standalone is configured in MC + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneName, mcName, true)).To(Succeed()) // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ################# Update Monitoring Console In Cluster Master CR ################################## - - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### + // Scale Standalone instance + testcaseEnvInst.Log.Info("Scaling Standalone CR") + scaledReplicaCount := 2 + standalone = &enterpriseApi.Standalone{} + Expect(testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone")).To(Succeed()) - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) + standalone.Spec.Replicas = int32(scaledReplicaCount) - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + err = deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Failed to scale Standalone") - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) + // Ensure standalone is scaling up + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseScalingUp)).To(Succeed()) - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Master Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // ################# Update Monitoring Console In SHC CR ################################## - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") - - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ############################ VERIFICATOIN FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) - - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // ############################ VERIFICATOIN FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) + // Wait for Standalone to be in READY status + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed()) - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) + // wait for custom resource resource version to change and verify MC is ready + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed()) - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false) + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + // Check both Standalone pods are configured in MC after scale up + testcaseEnvInst.Log.Info("Checking for Standalone Pods on MC after scale up") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true)).To(Succeed()) }) }) - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { - /* - Test Steps - 1. Deploy Multisite Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Master to use 2nd Monitoring Console - 9. Verify Cluster Master is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Master not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod - */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with SHC") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Master RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + Context("Standalone deployment (S1)", func() { + It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", func() { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, "search-head-adhoc", "search-head") + }) + }) - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) +}) - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) +// C3 reconfig and M4 tests — V3 (master) and V4 (manager) variants +var _ = Describe("Monitoring Console reconfig tests", func() { - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + // C3 reconfig tests + for _, cfg := range masterManagerMCConfigs { + cfg := cfg + Context("Clustered deployment C3 reconfig ("+cfg.Label+")", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, cfg.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(cfg.Label+", integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { + RunC3MCReconfigTest(ctx, deployment, testcaseEnvInst, cfg) + }) }) - }) + } + + // M4 reconfig tests + for _, cfg := range masterManagerMCConfigs { + cfg := cfg + Context("Multisite Clustered deployment M4 reconfig ("+cfg.Label+")", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, cfg.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(cfg.Label+", integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { + RunM4MCReconfigTest(ctx, deployment, testcaseEnvInst, cfg) + }) + }) + } }) diff --git a/test/monitoring_console/monitoring_console_test_shared.go b/test/monitoring_console/monitoring_console_test_shared.go new file mode 100644 index 000000000..35dc03950 --- /dev/null +++ b/test/monitoring_console/monitoring_console_test_shared.go @@ -0,0 +1,288 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package monitoringconsoletest + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunM4MCReconfigTest deploys an M4 multisite cluster with a Monitoring Console, +// verifies the MC configuration, then reconfigures the Cluster Manager to point +// to a second MC and verifies both MCs are updated correctly. +func RunM4MCReconfigTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg testenv.MCVersionConfig) { + defaultSHReplicas := 3 + defaultIndexerReplicas := 1 + siteCount := 3 + mcName := deployment.GetName() + + err := cfg.DeployM4WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) + Expect(err).To(Succeed(), "Unable to deploy multisite cluster") + + // Ensure cluster coordinator and all M4 components are ready + Expect(cfg.VerifyCMReady(ctx, deployment, testcaseEnvInst)).To(Succeed()) + Expect(testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount)).To(Succeed()) + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Generate pod name slices for verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), defaultIndexerReplicas, true, siteCount) + + // Verify MC configuration for M4 cluster + Expect(testenv.VerifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, mcName, shPods, indexerPods)).To(Succeed()) + + // ############ CLUSTER MANAGER MC RECONFIG ################################# + mcTwoName := deployment.GetName() + "-two" + cm := cfg.NewCMObject() + Expect(testcaseEnvInst.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, cm, deployment.GetName(), mcTwoName)).To(Succeed()) + + Expect(cfg.VerifyCMReady(ctx, deployment, testcaseEnvInst)).To(Succeed()) + + // Deploy and verify Monitoring Console Two + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + Expect(testenv.VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, shPods, indexerPods, false)).To(Succeed()) + Expect(testenv.VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcName, mc, shPods, true)).To(Succeed()) +} + +// RunC3MCReconfigTest deploys a C3 single-site cluster with a Monitoring Console, +// verifies the MC configuration, then reconfigures the Cluster Manager and SHC +// to point to a second MC and verifies both MCs are updated correctly. +func RunC3MCReconfigTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg testenv.MCVersionConfig) { + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + + // Deploy Monitoring Console Pod + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + err = cfg.DeployC3WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, func(ctx2 context.Context, d *testenv.Deployment) error { + return cfg.VerifyCMReady(ctx2, d, testcaseEnvInst) + })).To(Succeed()) + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed()) + + // Generate pod name slices for verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) + + // Verify MC configuration for C3 cluster + Expect(testenv.VerifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, mcName, shPods, indexerPods)).To(Succeed()) + + // Verify Monitoring Console is Ready and stays in ready state + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc)).To(Succeed()) + + // ################# Update Monitoring Console In Cluster Manager CR ################################## + + mcTwoName := deployment.GetName() + "-two" + cm := cfg.NewCMObject() + Expect(testcaseEnvInst.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, cm, deployment.GetName(), mcTwoName)).To(Succeed()) + + Expect(cfg.VerifyCMReady(ctx, deployment, testcaseEnvInst)).To(Succeed()) + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed()) + + // Deploy and verify Monitoring Console Two + mcTwo, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two") + + // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### + Expect(testenv.VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, shPods, indexerPods, true)).To(Succeed()) + + // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### + Expect(testenv.VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcName, mc, shPods, false)).To(Succeed()) + + // ################# Update Monitoring Console In SHC CR ################################## + + shc := &enterpriseApi.SearchHeadCluster{} + shcName := deployment.GetName() + "-shc" + Expect(testcaseEnvInst.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, shc, shcName, mcTwoName)).To(Succeed()) + + // Ensure Search Head Cluster goes to Ready Phase + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed()) + + if cfg.VerifyMCTwoReadyAfterSHC { + Expect(testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo)).To(Succeed()) + } + + // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### + Expect(testenv.VerifyMCTwoAfterSHCReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, shPods, indexerPods, cfg.SHCReconfigTimeout)).To(Succeed()) + + // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### + Expect(testenv.VerifyMCOneAfterSHCReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcName, mc, shPods, cfg.SHCReconfigTimeout)).To(Succeed()) +} + +// RunC3MCScaleUpTest deploys a C3 cluster with a Monitoring Console, verifies MC +// configuration, scales SHC and indexers, adds a standalone, and verifies the MC +// is updated correctly after scale up. Works for both V3 (master) and V4 (manager). +func RunC3MCScaleUpTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg testenv.MCVersionConfig) { + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + + // Deploy and verify Monitoring Console + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Deploy C3 cluster with MC + err = cfg.DeployC3WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Ensure C3 cluster is ready + Expect(testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, func(ctx2 context.Context, d *testenv.Deployment) error { + return cfg.VerifyCMReady(ctx2, d, testcaseEnvInst) + })).To(Succeed()) + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed()) + + // Verify MC configuration for C3 cluster + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) + Expect(testenv.VerifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, mcName, shPods, indexerPods)).To(Succeed()) + + // Scale Search Head Cluster + scaledSHReplicas := defaultSHReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + testcaseEnvInst.ScaleSearchHeadCluster(ctx, deployment, scaledSHReplicas) + + // Scale indexers + scaledIndexerReplicas := defaultIndexerReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + testcaseEnvInst.ScaleIndexerCluster(ctx, deployment, scaledIndexerReplicas) + + // Get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Deploy Standalone with MC reference + testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), mcName) + + // Ensure Indexer Cluster goes to Ready phase + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed()) + + // Ensure Search Head Cluster goes to Ready Phase + // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed()) + + // Wait for custom resource resource version to change and verify MC is ready + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed()) + + // Verify Standalone configured on Monitoring Console + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + Expect(testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, deployment.GetName(), mcName, true)).To(Succeed()) + + // Verify MC configuration after scale up + testcaseEnvInst.Log.Info("Verify MC configuration after Scale Up") + shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) + indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) + Expect(testenv.VerifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, mcName, shPods, indexerPods)).To(Succeed()) +} + +// RunS1StandaloneAddDeleteMCTest deploys two standalone instances with a Monitoring Console, +// verifies both are registered, then deletes the second standalone and verifies the MC +// config map and peer list are updated correctly. +func RunS1StandaloneAddDeleteMCTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, standaloneOneName, standaloneTwoName string) { + mcName := deployment.GetName() + + // Deploy Standalone one with MCRef + testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneOneName, mcName) + + // Deploy MC and wait for MC to be READY + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Check Standalone is configured in MC Config Map + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true)).To(Succeed()) + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Add another standalone instance in namespace + testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") + standaloneTwoSpec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("2"), + "memory": resource.MustParse("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("0.2"), + "memory": resource.MustParse("256Mi"), + }, + }, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone two to be in READY status + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo)).To(Succeed()) + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed()) + + // Check both standalones are configured in MC Config Map + standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true)).To(Succeed()) + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Delete standalone two and ensure MC is updated + testcaseEnvInst.Log.Info("Deleting second standalone deployment from namespace", "Standalone Name", standaloneTwoName) + deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) + err = deployment.DeleteCR(ctx, standaloneTwo) + Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed()) + + // Check standalone one is still configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true)).To(Succeed()) + + // Check Standalone Two NOT configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") + Expect(testenv.VerifyStandalonePodsInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, false)).To(Succeed()) +} diff --git a/test/secret/manager_secret_c3_test.go b/test/secret/manager_secret_c3_test.go deleted file mode 100644 index 6944755b4..000000000 --- a/test/secret/manager_secret_c3_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managersecret, smoke, c3: secret update on indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_m4_test.go b/test/secret/manager_secret_m4_test.go deleted file mode 100644 index a51dac966..000000000 --- a/test/secret/manager_secret_m4_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managersecret, integration, m4: secret update on multisite indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_s1_test.go b/test/secret/manager_secret_s1_test.go deleted file mode 100644 index 58a671b67..000000000 --- a/test/secret/manager_secret_s1_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("managersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("managersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_c3_test.go b/test/secret/secret_c3_test.go deleted file mode 100644 index 8baee517c..000000000 --- a/test/secret/secret_c3_test.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastersecret, smoke, c3: secret update on indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for ClusterInitialized event to confirm cluster is fully initialized - idxcName := deployment.GetName() + "-idxc" - err = testcaseEnvInst.WaitForClusterInitialized(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterInitialized event on IndexerCluster") - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on SearchHeadCluster - shcName := deployment.GetName() + "-shc" - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on IndexerCluster - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_m4_test.go b/test/secret/secret_m4_test.go deleted file mode 100644 index 3a1efe591..000000000 --- a/test/secret/secret_m4_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastersecret, integration, m4: secret update on multisite indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_s1_test.go b/test/secret/secret_s1_test.go deleted file mode 100644 index d680a4661..000000000 --- a/test/secret/secret_s1_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("mastersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("mastersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("mastersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_suite_test.go b/test/secret/secret_suite_test.go index c7d40946a..f7addae78 100644 --- a/test/secret/secret_suite_test.go +++ b/test/secret/secret_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package secret import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 10 * ConsistentPollInterval -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "secret-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -48,11 +37,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/secret/secret_test.go b/test/secret/secret_test.go new file mode 100644 index 000000000..3d6ed95e6 --- /dev/null +++ b/test/secret/secret_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// masterManagerConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 and M4 secret test tables. +var masterManagerConfigs = []testenv.MasterManagerTestConfig{ + {NamePrefix: "master", Label: "mastersecret", NewConfig: testenv.NewClusterReadinessConfigV3}, + {NamePrefix: "", Label: "managersecret", NewConfig: testenv.NewClusterReadinessConfigV4}, +} + +var _ = Describe("Secret test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + // S1 tests — both label variants use identical V4 config + s1SecretLabels := []string{"mastersecret", "managersecret"} + + for _, label := range s1SecretLabels { + label := label + Context("Standalone deployment (S1) with LM and MC", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(label+", integration, s1: Secret update on a standalone instance with LM and MC", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + + It(label+", integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteTest(ctx, deployment, testcaseEnvInst, config) + }) + + It(label+", smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteWithMCRefTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // C3 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerConfigs { + tc := tc + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", smoke, c3: secret update on Indexers and Search Head Cluster", func() { + config := tc.NewConfig() + RunC3SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // M4 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix, testenv.WithTimeout(40000)) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration, m4: secret update on multisite Indexers and Search Head Cluster", func() { + config := tc.NewConfig() + RunM4SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } +}) diff --git a/test/secret/secret_test_shared.go b/test/secret/secret_test_shared.go new file mode 100644 index 000000000..778e5b838 --- /dev/null +++ b/test/secret/secret_test_shared.go @@ -0,0 +1,133 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// RunS1SecretUpdateTest runs the standard S1 secret update test workflow +func RunS1SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + setup, err := testenv.SetupS1WithLMAndMC(ctx, deployment, testcaseEnvInst, config) + Expect(err).To(Succeed(), "Unable to setup S1 with LM and MC") + + // Update Secret Value on Secret Object + updatedSecretData, err := testenv.GenerateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, setup.NamespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to generate and apply secret update") + + Expect(testenv.VerifyS1SecretChangeApplied(ctx, deployment, testcaseEnvInst, config, setup, updatedSecretData, true)).To(Succeed()) +} + +// RunS1SecretDeleteTest runs the standard S1 secret delete test workflow +func RunS1SecretDeleteTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + setup, err := testenv.SetupS1WithLMAndMC(ctx, deployment, testcaseEnvInst, config) + Expect(err).To(Succeed(), "Unable to setup S1 with LM and MC") + + // Re-fetch secret struct so we can verify its data is restored after deletion + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), setup.NamespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete Secret Object + err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), setup.NamespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + Expect(testenv.VerifyS1SecretChangeApplied(ctx, deployment, testcaseEnvInst, config, setup, secretStruct.Data, false)).To(Succeed()) +} + +// RunS1SecretDeleteWithMCRefTest runs the S1 secret delete test with MC reference workflow +func RunS1SecretDeleteWithMCRefTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Create standalone Deployment with MonitoringConsoleRef + mcName := deployment.GetName() + standalone, err := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), mcName) + Expect(err).To(Succeed(), "Unable to deploy Standalone with MC reference") + + // Deploy and verify Monitoring Console + mc, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete secret by passing empty Data Map + err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + // Ensure standalone is updating + Expect(testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseUpdating)).To(Succeed()) + + // Wait for Standalone to be in READY status + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed()) + + Expect(testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion)).To(Succeed()) + + Expect(testenv.VerifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretStruct.Data, false)).To(Succeed()) +} + +// RunC3SecretUpdateTest runs the standard C3 secret update test workflow +func RunC3SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + mcRef := deployment.GetName() + Expect(config.DeployC3WithLicense(ctx, deployment, testcaseEnvInst, 3, true, mcRef)).To(Succeed()) + + mc, resourceVersion, updatedSecretData, err := testenv.ApplySecretUpdateAndVerifyCMUpdating(ctx, deployment, testcaseEnvInst, config) + Expect(err).To(Succeed(), "Unable to apply secret update and verify CM updating") + + Expect(testenv.VerifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config)).To(Succeed()) + + // Ensure Search Head Cluster goes to Ready phase + Expect(testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)).To(Succeed()) + + // Wait for PasswordSyncCompleted event on SearchHeadCluster + shcName := deployment.GetName() + "-shc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") + + // Ensure Indexers go to Ready phase + Expect(testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment)).To(Succeed()) + + // Wait for PasswordSyncCompleted event on IndexerCluster + idxcName := deployment.GetName() + "-idxc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") + + Expect(testenv.VerifyPostSecretChangeCluster(ctx, deployment, testcaseEnvInst, mc, resourceVersion, updatedSecretData)).To(Succeed()) +} + +// RunM4SecretUpdateTest runs the standard M4 secret update test workflow +func RunM4SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + siteCount := 3 + mcName := deployment.GetName() + + Expect(config.DeployM4WithLicense(ctx, deployment, testcaseEnvInst, 1, siteCount, mcName)).To(Succeed()) + + mc, resourceVersion, updatedSecretData, err := testenv.ApplySecretUpdateAndVerifyCMUpdating(ctx, deployment, testcaseEnvInst, config) + Expect(err).To(Succeed(), "Unable to apply secret update and verify CM updating") + + Expect(testenv.VerifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config)).To(Succeed()) + Expect(testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount)).To(Succeed()) + + Expect(testenv.VerifyPostSecretChangeCluster(ctx, deployment, testcaseEnvInst, mc, resourceVersion, updatedSecretData)).To(Succeed()) +} diff --git a/test/smartstore/manager_smartstore_test.go b/test/smartstore/manager_smartstore_test.go deleted file mode 100644 index f54a98da5..000000000 --- a/test/smartstore/manager_smartstore_test.go +++ /dev/null @@ -1,282 +0,0 @@ -package smartstore - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Smartstore test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure multiple indexes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure indexes which use default volumes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("managersmartstore, smoke: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-two" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } - }) - }) -}) diff --git a/test/smartstore/smartstore_suite_test.go b/test/smartstore/smartstore_suite_test.go index 693ca4cf8..d49cd9098 100644 --- a/test/smartstore/smartstore_suite_test.go +++ b/test/smartstore/smartstore_suite_test.go @@ -1,8 +1,20 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -10,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smartstore-" + testenv.RandomDNSName(3) @@ -26,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -35,11 +37,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/smartstore/smartstore_test.go b/test/smartstore/smartstore_test.go index bdd100f37..bb799a629 100644 --- a/test/smartstore/smartstore_test.go +++ b/test/smartstore/smartstore_test.go @@ -1,363 +1,108 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "context" - "fmt" "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) +// smartstoreTestConfig extends MasterManagerTestConfig with a per-variant +// timeout used by the S1 multiple-indexes test. +type smartstoreTestConfig struct { + testenv.MasterManagerTestConfig + S1IndexesTimeout time.Duration +} + +// masterManagerSmartstoreConfigs defines the V3 (master) and V4 (manager) variants +// shared by the S1 and M4 smartstore test tables. +var masterManagerSmartstoreConfigs = []smartstoreTestConfig{ + {testenv.MasterManagerTestConfig{NamePrefix: "master", Label: "mastersmartstore", NewConfig: testenv.NewClusterReadinessConfigV3}, 2 * time.Minute}, + {testenv.MasterManagerTestConfig{NamePrefix: "", Label: "managersmartstore", NewConfig: testenv.NewClusterReadinessConfigV4}, 5 * time.Minute}, +} + var _ = Describe("Smartstore test", func() { var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() var deployment *testenv.Deployment + ctx := context.TODO() - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure multiple indexes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure indexes which use default volumes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - + for _, tc := range masterManagerSmartstoreConfigs { + tc := tc + Context("Standalone deployment (S1)", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", integration: Can configure multiple indexes through app", func() { + RunS1MultipleIndexesTest(ctx, deployment, testcaseEnvInst, tc.S1IndexesTimeout) + }) + + It(tc.Label+", integration: Can configure indexes which use default volumes through app", func() { + RunS1DefaultVolumesTest(ctx, deployment, testcaseEnvInst) + }) }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("mastersmartstore, m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - oldBundleHash := testenv.GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify new bundle is pushed - testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, testcaseEnvInst.GetName(), 1, oldBundleHash) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } + Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + Expect(err).To(Succeed(), "Failed to setup test case environment") + }) + + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) + + It(tc.Label+", m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { + config := tc.NewConfig() + RunM4MultisiteSmartStoreTest(ctx, deployment, testcaseEnvInst, config) + }) }) - }) + } Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Etc storage", func() { - - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Etc and pvc for Var volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - EtcVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageEtcEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + BeforeEach(func() { + var err error + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "master") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Var storage", func() { - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Var and pvc for Etc volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - VarVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageVarEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + AfterEach(func() { + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") + }) - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Etc storage", func() { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageEtcEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, true) + }) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Var storage", func() { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageVarEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, false) }) }) }) diff --git a/test/smartstore/smartstore_test_shared.go b/test/smartstore/smartstore_test_shared.go new file mode 100644 index 000000000..2e5c7a5d6 --- /dev/null +++ b/test/smartstore/smartstore_test_shared.go @@ -0,0 +1,220 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package smartstore + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" +) + +// RunS1MultipleIndexesTest runs the standard S1 multiple indexes SmartStore test workflow +func RunS1MultipleIndexesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, waitTimeout time.Duration) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexVolumeMap := map[string]string{ + "test-index-" + testenv.RandomDNSName(3): volName, + "test-index-" + testenv.RandomDNSName(3): volName, + } + testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) + + var indexSpec []enterpriseApi.IndexSpec + volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + // Create index volume spec from index volume map + for index, volume := range indexVolumeMap { + indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) + } + + // Generate smartstore spec + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volumeSpec, + IndexList: indexSpec, + } + + // Deploy Standalone + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Wait for Standalone to reach Ready phase + err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, waitTimeout) + Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") + + // Verify standalone goes to ready state and stays ready + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed()) + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + for indexName := range indexVolumeMap { + Expect(testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName)).To(Succeed()) + } + + // Ingest data to the index + for indexName := range indexVolumeMap { + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, indexName, podName) + } + + // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 + for indexName := range indexVolumeMap { + testenv.RollHotToWarm(ctx, deployment, podName, indexName) + Expect(testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName)).To(Succeed()) + } +} + +// RunS1DefaultVolumesTest runs the standard S1 default volumes SmartStore test workflow +func RunS1DefaultVolumesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} + defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} + cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} + + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + Defaults: defaultSmartStoreSpec, + CacheManagerConf: cacheManagerSmartStoreSpec, + } + + // Deploy Standalone with given smartstore spec + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Verify standalone goes to ready state + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed()) + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + Expect(testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName)).To(Succeed()) + + // Check special index configs + Expect(testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"])).To(Succeed()) + + // Ingest data to the index + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(ctx, deployment, logFile, indexName, podName) + + // Roll Hot Buckets on the test index by restarting splunk + testenv.RollHotToWarm(ctx, deployment, podName, indexName) + + // Check for indexes on S3 + Expect(testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName)).To(Succeed()) + + // Verify Cachemanager Values + serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" + + // Validate MaxCacheSizeMB + Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB))).To(Succeed()) + + // Validate EvictionPaddingSizeMB + Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB))).To(Succeed()) + + // Validate MaxConcurrentDownloads + Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads))).To(Succeed()) + + // Validate MaxConcurrentUploads + Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads))).To(Succeed()) + + // Validate EvictionPolicy + Expect(testcaseEnvInst.VerifyConfOnPod(podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy)).To(Succeed()) +} + +// RunS1EphemeralStorageTest deploys a Standalone with one ephemeral storage volume configured and verifies it is ready. +// Pass etcStorage=true to set EtcVolumeStorageConfig, false to set VarVolumeStorageConfig. +func RunS1EphemeralStorageTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, storageConfig enterpriseApi.StorageClassSpec, etcStorage bool) { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + } + if etcStorage { + spec.CommonSplunkSpec.EtcVolumeStorageConfig = storageConfig + } else { + spec.CommonSplunkSpec.VarVolumeStorageConfig = storageConfig + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") + + Expect(testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone)).To(Succeed()) +} + +// RunM4MultisiteSmartStoreTest runs the standard M4 multisite SmartStore test workflow +func RunM4MultisiteSmartStoreTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + } + + siteCount := 3 + err := config.DeployMultisiteClusterWithIndexes(ctx, deployment, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + Expect(testenv.VerifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount)).To(Succeed()) + + // Use multisite workflow helper to verify index, ingest data, roll to warm, and verify on S3 + Expect(testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, siteCount, indexName)).To(Succeed()) + + // Get old bundle hash before adding new index + oldBundleHash := config.GetBundleHash(ctx, deployment) + + testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") + indexNameTwo := "test-index-" + testenv.RandomDNSName(3) + indexList := []string{indexName, indexNameTwo} + newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} + + // Update CR with new index based on API version + Expect(config.AppendSmartStoreIndex(ctx, deployment, newIndex)).To(Succeed(), "Unable to append SmartStore index") + + Expect(testenv.VerifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount)).To(Succeed()) + + // Verify new bundle is pushed (only for v3) + if config.GetAPIVersion() == "v3" { + Expect(testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, 1, oldBundleHash)).To(Succeed()) + } + + // Verify both indexes on all sites + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + for _, index := range indexList { + Expect(testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index)).To(Succeed()) + } + } + + // Use multisite workflow helper for the new index + testcaseEnvInst.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) + Expect(testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, siteCount, indexNameTwo)).To(Succeed()) +} diff --git a/test/smoke/cluster_manager_sites_response.go b/test/smoke/cluster_manager_sites_response.go deleted file mode 100644 index f9bcdee60..000000000 --- a/test/smoke/cluster_manager_sites_response.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package smoke - -// ClusterManagerSitesResponse is a representation of the sites managed by a Splunk cluster-manager -// Endpoint: /services/cluster/manager/sites -type ClusterManagerSitesResponse struct { - Entries []ClusterManagerSitesEntry `json:"entry"` -} - -// ClusterManagerSitesEntry represents a site of an indexer cluster with its metadata -type ClusterManagerSitesEntry struct { - Name string `json:"name"` - Content ClusterManagerSitesContent `json:"content"` -} - -// ClusterManagerSitesContent represents detailed information about a site -type ClusterManagerSitesContent struct { - Peers map[string]ClusterManagerSitesPeer `json:"peers"` -} - -// ClusterManagerSitesPeer reprensents an indexer peer member of a site -type ClusterManagerSitesPeer struct { - ServerName string `json:"server_name"` -} diff --git a/test/smoke/smoke_suite_test.go b/test/smoke/smoke_suite_test.go index b47f8a984..4242b7a38 100644 --- a/test/smoke/smoke_suite_test.go +++ b/test/smoke/smoke_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package smoke import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smoke-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -48,11 +37,11 @@ func TestBasic(t *testing.T) { var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(Succeed(), "Failed to initialize test environment") }) var _ = AfterSuite(func() { if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + Expect(testenvInstance.Teardown()).To(Succeed(), "Failed to teardown test environment") } }) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index d6d969819..63591cba7 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,16 +15,11 @@ package smoke import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - corev1 "k8s.io/api/core/v1" ) var _ = Describe("Smoke test", func() { @@ -35,133 +30,43 @@ var _ = Describe("Smoke test", func() { BeforeEach(func() { var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment, err = testenv.SetupTestCaseEnv(testenvInstance, "") + Expect(err).To(Succeed(), "Failed to setup test case environment") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Expect(testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment)).To(Succeed(), "Failed to teardown test case environment") }) Context("Standalone deployment (S1)", func() { It("smoke, basic, s1: can deploy a standalone instance", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + _, err := testcaseEnvInst.RunStandaloneDeploymentWorkflow(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, basic, c3: can deploy indexers and search head cluster", func() { - - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testcaseEnvInst.RunC3DeploymentWorkflow(ctx, deployment, 3, "") }) }) - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { It("smoke, basic, m4: can deploy indexers and search head cluster", func() { - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testcaseEnvInst.RunM4DeploymentWorkflow(ctx, deployment, 1, 3, "") }) }) - Context("Multisite cluster deployment (M1 - multisite indexer cluster)", func() { + Context("Multisite cluster deployment (M1 - Multisite Indexer Cluster)", func() { It("smoke, basic: can deploy multisite indexers cluster", func() { - - siteCount := 3 - err := deployment.DeployMultisiteCluster(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testcaseEnvInst.RunM1DeploymentWorkflow(ctx, deployment, 1, 3) }) }) Context("Standalone deployment (S1) with Service Account", func() { It("smoke, basic, s1: can deploy a standalone instance attached to a service account", func() { - // Create Service Account serviceAccountName := "smoke-service-account" - testcaseEnvInst.CreateServiceAccount(serviceAccountName) - - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ServiceAccount: serviceAccountName, - }, - } - - // Create standalone Deployment with License Manager - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify serviceAccount is configured on Pod - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(deployment, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName) + testcaseEnvInst.RunStandaloneWithServiceAccountWorkflow(ctx, deployment, serviceAccountName) }) }) }) diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index a1806d657..a8851c5cf 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -4,13 +4,11 @@ import ( "context" "errors" "fmt" - "os" "path/filepath" "strings" "sync" "time" - gomega "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" corev1 "k8s.io/api/core/v1" wait "k8s.io/apimachinery/pkg/util/wait" @@ -109,7 +107,7 @@ var PVTestAppsLocation = "appframework/100mb_sample_apps/" // AppStagingLocOnPod is the volume on Splunk pod where apps will be copied from operator var AppStagingLocOnPod = "/operator-staging/appframework/" -// AppDownloadVolume location on Operator pod for App framework downloads +// AppDownloadVolume location on Operator pod for App Framework downloads var AppDownloadVolume = "/opt/splunk/appframework/downloadedApps/" // GenerateAppSourceSpec return AppSourceSpec struct with given values @@ -125,21 +123,21 @@ func GenerateAppSourceSpec(appSourceName string, appSourceLocation string, appSo func GetPodAppStatus(ctx context.Context, deployment *Deployment, podName string, ns string, appname string, clusterWideInstall bool) (string, string, error) { // For clusterwide install do not check for versions on deployer and cluster-manager as the apps arent installed there if clusterWideInstall && (strings.Contains(podName, "-cluster-manager-") || strings.Contains(podName, splcommon.TestClusterManagerDashed) || strings.Contains(podName, "-deployer-")) { - logf.Log.Info("Pod skipped as install is Cluter-wide", "PodName", podName) + logf.Log.Info("Pod skipped as install is Cluter-wide", "podName", podName) return "", "", nil } - output, err := GetPodAppInstallStatus(ctx, deployment, podName, ns, appname) + output, err := GetPodAppInstallStatus(ctx, deployment, podName, appname) if err != nil { return "", "", err } status := strings.Fields(output)[2] - version, err := GetPodInstalledAppVersion(deployment, podName, ns, appname, clusterWideInstall) + version, err := GetPodInstalledAppVersion(podName, ns, appname, clusterWideInstall) return status, version, err } // GetPodInstalledAppVersion Get the version of the app installed on pod -func GetPodInstalledAppVersion(deployment *Deployment, podName string, ns string, appname string, clusterWideInstall bool) (string, error) { +func GetPodInstalledAppVersion(podName string, ns string, appname string, clusterWideInstall bool) (string, error) { path := "etc/apps" //For cluster-wide install the apps are extracted to different locations if clusterWideInstall { @@ -154,11 +152,11 @@ func GetPodInstalledAppVersion(deployment *Deployment, podName string, ns string } } filePath := fmt.Sprintf("/opt/splunk/%s/%s/default/app.conf", path, appname) - logf.Log.Info("Check app version", "App", appname, "Conf file", filePath) + logf.Log.Info("Check app version", "app", appname, "confFile", filePath) confline, err := GetConfLineFromPod(podName, filePath, ns, "version", "launcher", true) if err != nil { - logf.Log.Error(err, "Failed to get version from pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get version from pod", "podName", podName) return "", err } version := strings.TrimSpace(strings.Split(confline, "=")[1]) @@ -167,7 +165,7 @@ func GetPodInstalledAppVersion(deployment *Deployment, podName string, ns string } // GetPodAppInstallStatus Get the app install status -func GetPodAppInstallStatus(ctx context.Context, deployment *Deployment, podName string, ns string, appname string) (string, error) { +func GetPodAppInstallStatus(ctx context.Context, deployment *Deployment, podName string, appname string) (string, error) { stdin := fmt.Sprintf("/opt/splunk/bin/splunk display app '%s' -auth admin:$(cat /mnt/splunk-secrets/password)", appname) command := []string{"/bin/sh"} var stdout, stderr string @@ -181,31 +179,11 @@ func GetPodAppInstallStatus(ctx context.Context, deployment *Deployment, podName return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout) return strings.TrimSuffix(stdout, "\n"), nil } -// GetPodAppbtoolStatus Get the app btool status -func GetPodAppbtoolStatus(ctx context.Context, deployment *Deployment, podName string, ns string, appname string) (string, error) { - stdin := fmt.Sprintf("/opt/splunk/bin/splunk btool %s --app=%s --debug", appname, appname) - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command, "stdin", stdin) - return "", err - } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - - if len(stdout) > 0 { - if strings.Contains(strings.Split(stdout, "\n")[0], "App is disabled") { - return "DISABLED", nil - } - return "ENABLED", nil - } - return "", err -} - // GetAppFileList Get the Versioned App file list for app Names func GetAppFileList(appList []string) []string { appFileList := make([]string, 0, len(appList)) @@ -218,165 +196,213 @@ func GetAppFileList(appList []string) []string { // GetAppframeworkManualUpdateConfigMap gets config map for given manual update configmap func GetAppframeworkManualUpdateConfigMap(ctx context.Context, deployment *Deployment, ns string) (*corev1.ConfigMap, error) { ConfigMapName := fmt.Sprintf(AppframeworkManualUpdateConfigMap, ns) - logf.Log.Info("Get config map for", "CONFIG MAP NAME", ConfigMapName) + logf.Log.Info("Get config map for", "configMapName", ConfigMapName) ConfigMap, err := GetConfigMap(ctx, deployment, ns, ConfigMapName) if err != nil { logf.Log.Error(err, "Failed to get splunk manual poll Config Map") return ConfigMap, err } - logf.Log.Info("Config Map contents", "CONFIG MAP NAME", ConfigMapName, "Data", ConfigMap.Data) + logf.Log.Info("Config Map contents", "configMapName", ConfigMapName, "data", ConfigMap.Data) return ConfigMap, err } // GetAppDeploymentInfoStandalone returns AppDeploymentInfo for given standalone, appSourceName and appName -func GetAppDeploymentInfoStandalone(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoStandalone(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { standalone := &enterpriseApi.Standalone{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, standalone) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := standalone.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Standalone AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Standalone AppInfo Struct", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfoIngestorCluster returns AppDeploymentInfo for given IngestorCluster, appSourceName and appName -func GetAppDeploymentInfoIngestorCluster(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoIngestorCluster(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { ingestor := &enterpriseApi.IngestorCluster{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, ingestor) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := ingestor.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Ingestor AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Ingestor AppInfo Struct", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfoMonitoringConsole returns AppDeploymentInfo for given Monitoring Console, appSourceName and appName -func GetAppDeploymentInfoMonitoringConsole(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoMonitoringConsole(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { mc := &enterpriseApi.MonitoringConsole{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, mc) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := mc.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Monitoring Console AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Monitoring Console AppInfo Struct", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfoClusterManager returns AppDeploymentInfo for given Cluster Manager, appSourceName and appName -func GetAppDeploymentInfoClusterManager(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoClusterManager(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { cm := &enterpriseApi.ClusterManager{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Cluster Manager AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Cluster Manager AppInfo Struct", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } -// GetAppDeploymentInfoClusterMaster returns AppDeploymentInfo for given Cluster Manager, appSourceName and appName -func GetAppDeploymentInfoClusterMaster(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +// GetAppDeploymentInfoClusterMaster returns AppDeploymentInfo for given Cluster Master, appSourceName and appName +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoClusterMaster(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { cm := &enterpriseApiV3.ClusterMaster{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Cluster Master AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Cluster Master AppInfo Struct", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfoSearchHeadCluster returns AppDeploymentInfo for given Search Head Cluster, appSourceName and appName -func GetAppDeploymentInfoSearchHeadCluster(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoSearchHeadCluster(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { cm := &enterpriseApi.SearchHeadCluster{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Search Head Cluster AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Search Head Cluster AppInfo Struct", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appInfoList", appInfoList) + return appDeploymentInfo, err +} + +// GetAppDeploymentInfoLicenseManager returns AppDeploymentInfo for given License Manager, appSourceName and appName +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoLicenseManager(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { + lm := &enterpriseApi.LicenseManager{} + appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} + err := deployment.GetInstance(ctx, name, lm) + if err != nil { + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) + return appDeploymentInfo, err + } + appInfoList := lm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList + for _, appInfo := range appInfoList { + testenvInstance.Log.Info("Checking License Manager AppInfo Struct", "appName", appName, "appSource", appSourceName, "licenseManagerName", name, "appDeploymentInfo", appInfo) + if strings.Contains(appName, appInfo.AppName) { + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "licenseManagerName", name, "appDeploymentInfo", appInfo) + appDeploymentInfo = appInfo + return appDeploymentInfo, nil + } + } + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "licenseManagerName", name, "appInfoList", appInfoList) + return appDeploymentInfo, err +} + +// GetAppDeploymentInfoLicenseMaster returns AppDeploymentInfo for given License Master, appSourceName and appName +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfoLicenseMaster(ctx context.Context, deployment *Deployment, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { + lm := &enterpriseApiV3.LicenseMaster{} + appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} + err := deployment.GetInstance(ctx, name, lm) + if err != nil { + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) + return appDeploymentInfo, err + } + appInfoList := lm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList + for _, appInfo := range appInfoList { + testenvInstance.Log.Info("Checking License Master AppInfo Struct", "appName", appName, "appSource", appSourceName, "licenseMasterName", name, "appDeploymentInfo", appInfo) + if strings.Contains(appName, appInfo.AppName) { + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "licenseMasterName", name, "appDeploymentInfo", appInfo) + appDeploymentInfo = appInfo + return appDeploymentInfo, nil + } + } + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "licenseMasterName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } // GetAppDeploymentInfo returns AppDeploymentInfo for given CR Kind, appSourceName and appName -func GetAppDeploymentInfo(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { +func (testenvInstance *TestCaseEnv) GetAppDeploymentInfo(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { var appDeploymentInfo enterpriseApi.AppDeploymentInfo var err error switch crKind { case "Standalone": - appDeploymentInfo, err = GetAppDeploymentInfoStandalone(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoStandalone(ctx, deployment, name, appSourceName, appName) case "IngestorCluster": - appDeploymentInfo, err = GetAppDeploymentInfoIngestorCluster(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoIngestorCluster(ctx, deployment, name, appSourceName, appName) case "MonitoringConsole": - appDeploymentInfo, err = GetAppDeploymentInfoMonitoringConsole(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoMonitoringConsole(ctx, deployment, name, appSourceName, appName) case "SearchHeadCluster": - appDeploymentInfo, err = GetAppDeploymentInfoSearchHeadCluster(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoSearchHeadCluster(ctx, deployment, name, appSourceName, appName) case "ClusterManager": - appDeploymentInfo, err = GetAppDeploymentInfoClusterManager(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoClusterManager(ctx, deployment, name, appSourceName, appName) case "ClusterMaster": - appDeploymentInfo, err = GetAppDeploymentInfoClusterMaster(ctx, deployment, testenvInstance, name, appSourceName, appName) + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoClusterMaster(ctx, deployment, name, appSourceName, appName) + case "LicenseManager": + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoLicenseManager(ctx, deployment, name, appSourceName, appName) + case "LicenseMaster": + appDeploymentInfo, err = testenvInstance.GetAppDeploymentInfoLicenseMaster(ctx, deployment, name, appSourceName, appName) default: message := fmt.Sprintf("Failed to fetch AppDeploymentInfo. Incorrect CR Kind %s", crKind) err = errors.New(message) @@ -386,26 +412,8 @@ func GetAppDeploymentInfo(ctx context.Context, deployment *Deployment, testenvIn } // GenerateAppFrameworkSpec Generate Appframework spec -func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeName string, scope string, appSourceName string, s3TestDir string, pollInterval int) enterpriseApi.AppFrameworkSpec { - var volumeSpec []enterpriseApi.VolumeSpec - - // Create App framework volume - switch ClusterProvider { - case "eks": - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3", GetDefaultS3Region())} - case "azure": - managedID := os.Getenv("AZURE_MANAGED_ID_ENABLED") - if managedID == "false" { - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzure(volumeName, GetAzureEndpoint(ctx), testenvInstance.GetIndexSecretName(), "azure", "blob")} - } else { - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob")} - } - case "gcp": - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultS3Region())} - - default: - testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") - } +func (testenvInstance *TestCaseEnv) GenerateAppFrameworkSpec(ctx context.Context, volumeName string, scope string, appSourceName string, s3TestDir string, pollInterval int) enterpriseApi.AppFrameworkSpec { + volumeSpec := testenvInstance.GenerateVolumeSpecForProvider(ctx, volumeName) // AppSourceDefaultSpec: Remote Storage volume name and Scope of App deployment appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -429,17 +437,15 @@ func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, // WaitforPhaseChange Wait for timeout or when phase change is seen on a CR for any particular app // Deprecated: Use WaitForAppPhaseChange instead for better timeout control -func WaitforPhaseChange(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string, appSourceName string, appList []string) { - if err := WaitForAppPhaseChange(ctx, deployment, testenvInstance, name, crKind, appSourceName, appList, 2*time.Minute); err != nil { - testenvInstance.Log.Error(err, "WaitforPhaseChange did not observe a phase transition within timeout", "cr", name, "kind", crKind, "appSource", appSourceName) - } +func (testenvInstance *TestCaseEnv) WaitforPhaseChange(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string) { + _ = testenvInstance.WaitForAppPhaseChange(ctx, deployment, name, crKind, appSourceName, appList, 2*time.Minute) } // WaitForAppPhaseChange waits for any app in the list to change from PhaseInstall to another phase -func WaitForAppPhaseChange(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string, appSourceName string, appList []string, timeout time.Duration) error { +func (testenvInstance *TestCaseEnv) WaitForAppPhaseChange(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string, timeout time.Duration) error { return wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { for _, appName := range appList { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenvInstance, name, crKind, appSourceName, appName) + appDeploymentInfo, err := testenvInstance.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) if err != nil { testenvInstance.Log.Error(err, "Failed to get app deployment info") continue @@ -453,7 +459,7 @@ func WaitForAppPhaseChange(ctx context.Context, deployment *Deployment, testenvI } // VerifyAppFrameworkState will perform several verifications needed between the different steps of App Framework tests -func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deployment *Deployment, appSource []AppSourceInfo, splunkPodAge map[string]string, clusterManagerBundleHash string) string { +func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deployment *Deployment, appSource []AppSourceInfo, splunkPodAge map[string]string, clusterManagerBundleHash string) (string, error) { /* Function Steps * Verify apps 'download' and 'podCopy' states for all CRs (PARALLELIZED) * Verify apps packages are deleted from the operator pod for all CRs @@ -492,20 +498,24 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym wg.Wait() close(errChan) + var phaseErrs []error for err := range errChan { if err != nil { testenv.Log.Error(err, "Parallel phase verification failed") - gomega.Expect(err).To(gomega.Succeed()) + phaseErrs = append(phaseErrs, err) } } + if len(phaseErrs) > 0 { + return "", fmt.Errorf("parallel download/podCopy phase verification failed: %w", errors.Join(phaseErrs...)) + } testenv.Log.Info("Parallel verification of app download and podCopy phases completed successfully") // Verify apps packages are deleted from the operator pod for all CRs - opPod := GetOperatorPodName(testenv) + opPod := testenv.GetOperatorPodName() for _, appSource := range appSource { testenv.Log.Info(fmt.Sprintf("Verify apps %s packages are deleted from the operator pod for CR %v with name %v", appSource.CrAppVersion, appSource.CrKind, appSource.CrName)) opPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenv.GetName(), appSource.CrKind, deployment.GetName(), appSource.CrAppScope, appSource.CrAppSourceName) - testenv.VerifyAppsPackageDeletedOnOperatorContainer(ctx, deployment, testenv.GetName(), []string{opPod}, appSource.CrAppFileList, opPath) + testenv.VerifyAppsPackageDeletedOnOperatorContainer(ctx, deployment, []string{opPod}, appSource.CrAppFileList, opPath) } // Verify apps 'install' state for all CRs IN PARALLEL @@ -534,12 +544,16 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym wg.Wait() close(errChan) + var installErrs []error for err := range errChan { if err != nil { testenv.Log.Error(err, "Parallel install phase verification failed") - gomega.Expect(err).To(gomega.Succeed()) + installErrs = append(installErrs, err) } } + if len(installErrs) > 0 { + return "", fmt.Errorf("parallel install phase verification failed: %w", errors.Join(installErrs...)) + } testenv.Log.Info("Parallel verification of app install phase completed successfully") // Verify apps packages are deleted from the CR pods @@ -547,14 +561,14 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym podDownloadPath := AppStagingLocOnPod + appSource.CrAppSourceVolumeName pod := appSource.CrPod testenv.Log.Info(fmt.Sprintf("Verify %s apps packages are deleted on pod %s", appSource.CrAppVersion, pod)) - testenv.VerifyAppsPackageDeletedOnContainer(ctx, deployment, testenv.GetName(), pod, appSource.CrAppFileList, podDownloadPath) + testenv.VerifyAppsPackageDeletedOnContainer(ctx, deployment, pod, appSource.CrAppFileList, podDownloadPath) } // Verify bundle push status for _, appSource := range appSource { if (appSource.CrKind == "ClusterManager" || appSource.CrKind == "ClusterMaster") && appSource.CrAppScope == enterpriseApi.ScopeCluster { testenv.Log.Info(fmt.Sprintf("Verify Cluster Manager bundle push status (%s apps) and compare bundle hash with previous bundle hash", appSource.CrAppVersion)) - testenv.VerifyClusterManagerBundlePush(ctx, deployment, testenv.GetName(), appSource.CrReplicas, clusterManagerBundleHash) + testenv.VerifyClusterManagerBundlePush(ctx, deployment, appSource.CrReplicas, clusterManagerBundleHash) if clusterManagerBundleHash == "" { clusterManagerBundleHash = GetClusterManagerBundleHash(ctx, deployment, appSource.CrKind) } @@ -569,12 +583,12 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym for _, appSource := range appSource { if appSource.CrAppScope == enterpriseApi.ScopeLocal { testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'local' scope are copied to /etc/apps/ for CR %s with name %s", appSource.CrAppVersion, appSource.CrKind, appSource.CrName)) - testenv.VerifyAppsCopied(ctx, deployment, testenv.GetName(), appSource.CrPod, appSource.CrAppList, true, appSource.CrAppScope) + testenv.VerifyAppsCopied(ctx, deployment, appSource.CrPod, appSource.CrAppList, true, appSource.CrAppScope) } else { testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'cluster' scope are NOT copied to /etc/apps/ on %v pod", appSource.CrAppVersion, appSource.CrPod)) - testenv.VerifyAppsCopied(ctx, deployment, testenv.GetName(), appSource.CrPod, appSource.CrAppList, false, appSource.CrAppScope) + testenv.VerifyAppsCopied(ctx, deployment, appSource.CrPod, appSource.CrAppList, false, appSource.CrAppScope) testenv.Log.Info(fmt.Sprintf("Verify %s apps with 'cluster' scope are copied on %v pods", appSource.CrAppVersion, appSource.CrClusterPods)) - testenv.VerifyAppsCopied(ctx, deployment, testenv.GetName(), appSource.CrClusterPods, appSource.CrAppList, true, appSource.CrAppScope) + testenv.VerifyAppsCopied(ctx, deployment, appSource.CrClusterPods, appSource.CrAppList, true, appSource.CrAppScope) } } @@ -591,11 +605,11 @@ func (testenv *TestCaseEnv) VerifyAppFrameworkState(ctx context.Context, deploym testenv.VerifyAppInstalled(ctx, deployment, testenv.GetName(), allPodNames, appSource.CrAppList, true, "enabled", checkUpdated, true) } } - return clusterManagerBundleHash + return clusterManagerBundleHash, nil } // GetIsDeploymentInProgressFlag returns IsDeploymentInProgress for given CR Name, CR Kind -func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string) (bool, error) { +func (testenvInstance *TestCaseEnv) GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, name string, crKind string) (bool, error) { var isDeploymentInProgress bool var err error switch crKind { @@ -603,7 +617,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.Standalone{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -611,7 +625,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.MonitoringConsole{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -619,7 +633,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.SearchHeadCluster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -627,7 +641,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApiV3.ClusterMaster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -635,7 +649,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.ClusterManager{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -643,7 +657,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApiV3.LicenseMaster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -651,7 +665,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.LicenseManager{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress diff --git a/test/testenv/assertion_helpers.go b/test/testenv/assertion_helpers.go new file mode 100644 index 000000000..a28ee5b29 --- /dev/null +++ b/test/testenv/assertion_helpers.go @@ -0,0 +1,213 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// ScaleSearchHeadCluster scales a Search Head Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleSearchHeadCluster(ctx context.Context, deployment *Deployment, newReplicas int) error { + shcName := deployment.GetName() + "-shc" + + // Get instance of current SHC CR with latest config + shc := &enterpriseApi.SearchHeadCluster{} + if err := deployment.GetInstance(ctx, shcName, shc); err != nil { + return fmt.Errorf("failed to get instance of Search Head Cluster: %w", err) + } + + // Update Replicas of SHC + shc.Spec.Replicas = int32(newReplicas) + if err := deployment.UpdateCR(ctx, shc); err != nil { + return fmt.Errorf("failed to scale Search Head Cluster: %w", err) + } + + // Verify Search Head Cluster scales up and goes to ScalingUp phase + return testcaseenv.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) +} + +// ScaleIndexerCluster scales an Indexer Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleIndexerCluster(ctx context.Context, deployment *Deployment, newReplicas int) error { + idxcName := deployment.GetName() + "-idxc" + + // Get instance of current Indexer CR with latest config + idxc := &enterpriseApi.IndexerCluster{} + if err := deployment.GetInstance(ctx, idxcName, idxc); err != nil { + return fmt.Errorf("failed to get instance of Indexer Cluster: %w", err) + } + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(newReplicas) + if err := deployment.UpdateCR(ctx, idxc); err != nil { + return fmt.Errorf("failed to scale Indexer Cluster: %w", err) + } + + // Verify Indexer Cluster scales up and goes to ScalingUp phase + return testcaseenv.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) +} + +// UpdateMonitoringConsoleRefAndVerify updates the MonitoringConsoleRef in a CR and waits for the change to apply +func (testcaseenv *TestCaseEnv) UpdateMonitoringConsoleRefAndVerify(ctx context.Context, deployment *Deployment, obj interface{}, instanceName string, newMCName string) error { + // Get current resource version before update + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, obj) + + // Update the MonitoringConsoleRef based on the type + switch cr := obj.(type) { + case *enterpriseApi.ClusterManager: + if err := deployment.GetInstance(ctx, instanceName, cr); err != nil { + return fmt.Errorf("failed to get instance %s: %w", instanceName, err) + } + cr.Spec.MonitoringConsoleRef.Name = newMCName + if err := deployment.UpdateCR(ctx, cr); err != nil { + return fmt.Errorf("failed to update MonitoringConsoleRef: %w", err) + } + case *enterpriseApiV3.ClusterMaster: + if err := deployment.GetInstance(ctx, instanceName, cr); err != nil { + return fmt.Errorf("failed to get instance %s: %w", instanceName, err) + } + cr.Spec.MonitoringConsoleRef.Name = newMCName + if err := deployment.UpdateCR(ctx, cr); err != nil { + return fmt.Errorf("failed to update MonitoringConsoleRef: %w", err) + } + case *enterpriseApi.SearchHeadCluster: + if err := deployment.GetInstance(ctx, instanceName, cr); err != nil { + return fmt.Errorf("failed to get instance %s: %w", instanceName, err) + } + cr.Spec.MonitoringConsoleRef.Name = newMCName + if err := deployment.UpdateCR(ctx, cr); err != nil { + return fmt.Errorf("failed to update MonitoringConsoleRef: %w", err) + } + } + + // Wait for custom resource version to change + return testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, obj, resourceVersion) +} + +// DeployStandaloneWithMCRef deploys a standalone instance with a MonitoringConsoleRef +func (testcaseenv *TestCaseEnv) DeployStandaloneWithMCRef(ctx context.Context, deployment *Deployment, deploymentName string, mcName string) (*enterpriseApi.Standalone, error) { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseenv.GetSplunkImage(), + }, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deploymentName, spec) + if err != nil { + return nil, fmt.Errorf("unable to deploy standalone instance: %w", err) + } + + // Wait for Standalone to be in READY status + if err = testcaseenv.VerifyStandaloneReady(ctx, deployment, deploymentName, standalone); err != nil { + return nil, fmt.Errorf("standalone not ready: %w", err) + } + + return standalone, nil +} + +// VerifyStandaloneInMC verifies that a standalone instance is configured in the MC +func (testcaseenv *TestCaseEnv) VerifyStandaloneInMC(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shouldExist bool) error { + standalonePod := fmt.Sprintf(StandalonePod, deploymentName, 0) + if err := testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{standalonePod}, "SPLUNK_STANDALONE_URL", mcName, shouldExist); err != nil { + return err + } + return testcaseenv.VerifyPodsInMCConfigString(ctx, []string{standalonePod}, mcName, shouldExist, false) +} + +// VerifyLMConfiguredOnPods verifies License Manager is configured on all given pods +func VerifyLMConfiguredOnPods(ctx context.Context, deployment *Deployment, podNames []string) error { + for _, podName := range podNames { + if err := VerifyLMConfiguredOnPod(ctx, deployment, podName); err != nil { + return err + } + } + return nil +} + +// VerifyM1ClusterReady verifies the cluster coordinator, indexers, and multisite status are ready (no SHC). +func (testcaseenv *TestCaseEnv) VerifyM1ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment) error) error { + if err := verifyCoordinator(ctx, deployment); err != nil { + return err + } + if err := testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + return testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) +} + +// VerifyM4ClusterReady verifies the cluster coordinator, indexers, multisite status, and SHC are ready. +func (testcaseenv *TestCaseEnv) VerifyM4ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment) error) error { + if err := verifyCoordinator(ctx, deployment); err != nil { + return err + } + if err := testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + if err := testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount); err != nil { + return err + } + return testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyM4IndexersAndSHCReady verifies the cluster coordinator, indexers, and SHC are ready (without multisite check). +func (testcaseenv *TestCaseEnv) VerifyM4IndexersAndSHCReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment) error) error { + if err := verifyCoordinator(ctx, deployment); err != nil { + return err + } + if err := testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + return testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyC3ClusterReady verifies the cluster coordinator, SHC, and single-site indexers are ready. +func (testcaseenv *TestCaseEnv) VerifyC3ClusterReady(ctx context.Context, deployment *Deployment, verifyCoordinator func(context.Context, *Deployment) error) error { + if err := verifyCoordinator(ctx, deployment); err != nil { + return err + } + if err := testcaseenv.VerifySearchHeadClusterReady(ctx, deployment); err != nil { + return err + } + return testcaseenv.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// IngestDataOnIndexers ingests test data on all indexer pods +func IngestDataOnIndexers(ctx context.Context, deployment *Deployment, indexerCount int) { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deployment.GetName(), i) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, deployment, logFile, DefaultIngestIndex, podName) + } +} + +// IngestDataOnMultisiteIndexers ingests test data on all multisite indexer pods +func IngestDataOnMultisiteIndexers(ctx context.Context, deployment *Deployment, siteCount int) { + for site := 1; site <= siteCount; site++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), site, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, deployment, logFile, DefaultIngestIndex, podName) + } +} diff --git a/test/testenv/azureutils.go b/test/testenv/azureutils.go index 84a3f0601..04491d808 100644 --- a/test/testenv/azureutils.go +++ b/test/testenv/azureutils.go @@ -338,7 +338,7 @@ func (client *AzureBlobClient) DownloadFileFromAzure(ctx context.Context, downlo return "", err } - logf.Log.Info("Download from Azure successful:", "File", downloadRequest.RemoteFile) + logf.Log.Info("Download from Azure successful:", "file", downloadRequest.RemoteFile) return localFile.Name(), err } @@ -354,7 +354,7 @@ func DownloadFilesFromAzure(ctx context.Context, endPoint, accountKey, accountNa } _, err := azureBlobClient.DownloadFileFromAzure(ctx, downloadRequest, endPoint, StorageAccountKey, StorageAccount) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -374,7 +374,7 @@ func DownloadLicenseFromAzure(ctx context.Context, downloadDir string) (string, azureBlobClient := &AzureBlobClient{} filename, err := azureBlobClient.DownloadFileFromAzure(ctx, downloadRequest, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount) if err != nil { - logf.Log.Error(err, "Unable to download license file", "File", filename) + logf.Log.Error(err, "Unable to download license file", "file", filename) } return filename, err } @@ -431,7 +431,7 @@ func UploadFilesToAzure(ctx context.Context, accountName, accountKey, uploadFrom fileFullPath := "https://" + StorageAccount + ".blob.core.windows.net" + "/" + azureIndexesContainer + "/" + containerName + "/" + key fileName, err := UploadFileToAzure(ctx, accountName, accountKey, fileFullPath, fileLocation) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } uploadedFiles = append(uploadedFiles, fileName) diff --git a/test/testenv/cmutil.go b/test/testenv/cmutil.go index a1b71f3ce..f811aab7b 100644 --- a/test/testenv/cmutil.go +++ b/test/testenv/cmutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +44,7 @@ type ClusterManagerSitesContent struct { Peers map[string]ClusterManagerSitesPeer `json:"peers"` } -// ClusterManagerSitesPeer reprensents an indexer peer member of a site +// ClusterManagerSitesPeer represents an indexer peer member of a site type ClusterManagerSitesPeer struct { ServerName string `json:"server_name"` } @@ -73,13 +73,18 @@ type ClusterManagerHealthContent struct { SiteSearchFactorMet string `json:"site_search_factor_met"` } -// CheckRFSF check if cluster has met replication factor and search factor -func CheckRFSF(ctx context.Context, deployment *Deployment) bool { - //code to execute - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") +// GetCMPodName returns the cluster-manager or cluster-master pod name based on +// whether the deployment name contains "master". +func GetCMPodName(deployment *Deployment) string { if strings.Contains(deployment.GetName(), "master") { - podName = fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-master") + return fmt.Sprintf(ClusterMasterPod, deployment.GetName()) } + return fmt.Sprintf(ClusterManagerPod, deployment.GetName()) +} + +// CheckRFSF check if cluster has met replication factor and search factor +func CheckRFSF(ctx context.Context, deployment *Deployment) bool { + podName := GetCMPodName(deployment) stdin := "curl -ks -u admin:$(cat /mnt/splunk-secrets/password) https://localhost:8089/services/cluster/manager/health?output_mode=json" command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) @@ -132,11 +137,7 @@ func GetIndexersOrSearchHeadsOnCM(ctx context.Context, deployment *Deployment, e } else { url = "https://localhost:8089/services/cluster/manager/peers?output_mode=json" } - //code to execute - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - if strings.Contains(deployment.name, "master") { - podName = fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-master") - } + podName := GetCMPodName(deployment) stdin := fmt.Sprintf("curl -ks -u admin:$(cat /mnt/splunk-secrets/password) %s", url) command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) @@ -158,7 +159,7 @@ func CheckIndexerOnCM(ctx context.Context, deployment *Deployment, indexerName s restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "peer") found := false for _, entry := range restResponse.Entry { - logf.Log.Info("Peer found On CM", "Indexer Name", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Peer found On CM", "indexerName", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Label == indexerName { found = true break @@ -172,7 +173,7 @@ func CheckSearchHeadOnCM(ctx context.Context, deployment *Deployment, searchHead restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh") found := false for _, entry := range restResponse.Entry { - logf.Log.Info("Search Head On CM", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Search Head On CM", "searchHead", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Label == searchHeadName { found = true break @@ -186,7 +187,7 @@ func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh") searchHeadRemoved := true for _, entry := range restResponse.Entry { - logf.Log.Info("Search Found", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Search Found", "searchHead", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Status == "Disconnected" { searchHeadRemoved = false } @@ -194,21 +195,6 @@ func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { return searchHeadRemoved } -// RollHotBuckets roll hot buckets in cluster -func RollHotBuckets(ctx context.Context, deployment *Deployment) bool { - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - stdin := "/opt/splunk/bin/splunk rolling-restart cluster-peers -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - - return strings.Contains(stdout, "Rolling restart of all cluster peers has been initiated.") -} - // ClusterManagerInfoEndpointResponse is represtentation of /services/cluster/manager/info endpoint type ClusterManagerInfoEndpointResponse struct { Entry []struct { @@ -244,21 +230,8 @@ func ClusterManagerInfoResponse(ctx context.Context, deployment *Deployment, pod // CheckRollingRestartStatus checks if rolling restart is happening in cluster func CheckRollingRestartStatus(ctx context.Context, deployment *Deployment) bool { - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - stdin := "curl -ks -u admin:$(cat /mnt/splunk-secrets/password) https://localhost:8089/services/cluster/manager/info?output_mode=json" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - restResponse := ClusterManagerInfoEndpointResponse{} - err = json.Unmarshal([]byte(stdout), &restResponse) - if err != nil { - logf.Log.Error(err, "Failed to parse cluster searchheads") - return false - } + podName := GetCMPodName(deployment) + restResponse := ClusterManagerInfoResponse(ctx, deployment, podName) rollingRestart := true for _, entry := range restResponse.Entry { rollingRestart = entry.Content.RollingRestartFlag @@ -275,7 +248,7 @@ func CMBundlePushstatus(ctx context.Context, deployment *Deployment, previousBun // Check if new bundle was pushed by comparing hash if previousBundleHash != "" { if entry.Content.BundleID == previousBundleHash { - logf.Log.Info("Bundle hash not updated", "old Bundle hash", previousBundleHash, "new Bundle hash", entry.Content.BundleID) + logf.Log.Info("Bundle hash not updated", "oldBundleHash", previousBundleHash, "newBundleHash", entry.Content.BundleID) continue } } @@ -298,6 +271,6 @@ func GetClusterManagerBundleHash(ctx context.Context, deployment *Deployment, cr restResponse := ClusterManagerInfoResponse(ctx, deployment, podName) bundleHash := restResponse.Entry[0].Content.ActiveBundle.Checksum - logf.Log.Info("Bundle Hash on Cluster Manager Found", "Hash", bundleHash) + logf.Log.Info("Bundle Hash on Cluster Manager Found", "hash", bundleHash) return bundleHash } diff --git a/test/testenv/common_test_patterns.go b/test/testenv/common_test_patterns.go new file mode 100644 index 000000000..c07f5563b --- /dev/null +++ b/test/testenv/common_test_patterns.go @@ -0,0 +1,375 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/splunk/enterprise" + corev1 "k8s.io/api/core/v1" +) + +// ClusterCoordinator abstracts the v3/v4 API differences for cluster +// manager and license manager operations. +type ClusterCoordinator interface { + LicenseManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error + ClusterManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error + DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) + DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error + DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error + VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error + DeleteClusterManager(ctx context.Context, deployment *Deployment) error + AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) error + GetBundleHash(ctx context.Context, deployment *Deployment) string + ClusterManagerPVCType() string + GetAPIVersion() string +} + +// clusterMasterCoordinator implements ClusterCoordinator for v3 (ClusterMaster/LicenseMaster). +type clusterMasterCoordinator struct{} + +func (c *clusterMasterCoordinator) LicenseManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) +} + +func (c *clusterMasterCoordinator) ClusterManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyClusterMasterReady(ctx, deployment) +} + +func (c *clusterMasterCoordinator) DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) { + return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) +} + +func (c *clusterMasterCoordinator) DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error { + return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) +} + +func (c *clusterMasterCoordinator) DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { + return deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) +} + +func (c *clusterMasterCoordinator) VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) +} + +func (c *clusterMasterCoordinator) DeleteClusterManager(ctx context.Context, deployment *Deployment) error { + return GetAndDeleteCR(ctx, deployment, &enterpriseApiV3.ClusterMaster{}, deployment.GetName()) +} + +func (c *clusterMasterCoordinator) AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) error { + name := deployment.GetName() + cm := &enterpriseApiV3.ClusterMaster{} + if err := deployment.GetInstance(ctx, name, cm); err != nil { + return fmt.Errorf("failed to get instance of Cluster Master: %w", err) + } + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + if err := deployment.UpdateCR(ctx, cm); err != nil { + return fmt.Errorf("failed to add new index to Cluster Master: %w", err) + } + return nil +} + +func (c *clusterMasterCoordinator) GetBundleHash(ctx context.Context, deployment *Deployment) string { + return GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") +} + +func (c *clusterMasterCoordinator) ClusterManagerPVCType() string { + return "cluster-master" +} + +func (c *clusterMasterCoordinator) GetAPIVersion() string { + return "v3" +} + +// clusterManagerCoordinator implements ClusterCoordinator for v4 (ClusterManager/LicenseManager). +type clusterManagerCoordinator struct{} + +func (c *clusterManagerCoordinator) LicenseManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) +} + +func (c *clusterManagerCoordinator) ClusterManagerReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyClusterManagerReady(ctx, deployment) +} + +func (c *clusterManagerCoordinator) DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) { + return deployment.DeployStandaloneWithLM(ctx, name, mcRef) +} + +func (c *clusterManagerCoordinator) DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error { + return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) +} + +func (c *clusterManagerCoordinator) DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { + return deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) +} + +func (c *clusterManagerCoordinator) VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) +} + +func (c *clusterManagerCoordinator) DeleteClusterManager(ctx context.Context, deployment *Deployment) error { + return GetAndDeleteCR(ctx, deployment, &enterpriseApi.ClusterManager{}, deployment.GetName()) +} + +func (c *clusterManagerCoordinator) AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) error { + name := deployment.GetName() + cm := &enterpriseApi.ClusterManager{} + if err := deployment.GetInstance(ctx, name, cm); err != nil { + return fmt.Errorf("failed to get instance of Cluster Manager: %w", err) + } + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + if err := deployment.UpdateCR(ctx, cm); err != nil { + return fmt.Errorf("failed to add new index to Cluster Manager: %w", err) + } + return nil +} + +func (c *clusterManagerCoordinator) GetBundleHash(ctx context.Context, deployment *Deployment) string { + return GetClusterManagerBundleHash(ctx, deployment, "ClusterManager") +} + +func (c *clusterManagerCoordinator) ClusterManagerPVCType() string { + return "cluster-manager" +} + +func (c *clusterManagerCoordinator) GetAPIVersion() string { + return "v4" +} + +// ClusterReadinessConfig embeds a ClusterCoordinator and provides composed +// deployment and verification workflows shared across test packages. +type ClusterReadinessConfig struct { + ClusterCoordinator +} + +// MasterManagerTestConfig pairs a name prefix and test label with a factory +// function that returns the appropriate ClusterReadinessConfig. +// This is the standard config type shared by test packages that loop over +// V3 (master) and V4 (manager) variants. +type MasterManagerTestConfig struct { + NamePrefix string + Label string + NewConfig func() *ClusterReadinessConfig +} + +// NewClusterReadinessConfigV3 creates a ClusterReadinessConfig for v3 API (LicenseMaster/ClusterMaster) +func NewClusterReadinessConfigV3() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ClusterCoordinator: &clusterMasterCoordinator{}} +} + +// NewClusterReadinessConfigV4 creates a ClusterReadinessConfig for v4 API (LicenseManager/ClusterManager) +func NewClusterReadinessConfigV4() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ClusterCoordinator: &clusterManagerCoordinator{}} +} + +// VerifyC3ClusterReady verifies the C3 cluster is ready using the config's ClusterManagerReady callback. +func (c *ClusterReadinessConfig) VerifyC3ClusterReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) error { + return testcaseEnv.VerifyC3ClusterReady(ctx, deployment, func(ctx context.Context, d *Deployment) error { + return c.ClusterManagerReady(ctx, d, testcaseEnv) + }) +} + +// DeployMCAndGetVersion deploys and verifies a Monitoring Console, then returns both the MC +// instance and its current resource version. +func (testcaseenv *TestCaseEnv) DeployMCAndGetVersion(ctx context.Context, deployment *Deployment, name string, lmRef string) (*enterpriseApi.MonitoringConsole, string, error) { + mc, err := testcaseenv.DeployAndVerifyMonitoringConsole(ctx, deployment, name, lmRef) + if err != nil { + return nil, "", err + } + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, mc) + return mc, resourceVersion, nil +} + +// DeployAndVerifyStandalone deploys a standalone instance and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyStandalone(ctx context.Context, deployment *Deployment, mcRef string, licenseManagerRef string) (*enterpriseApi.Standalone, error) { + name := deployment.GetName() + standalone, err := deployment.DeployStandalone(ctx, name, mcRef, licenseManagerRef) + if err != nil { + return nil, fmt.Errorf("unable to deploy Standalone instance: %w", err) + } + if err := testcaseenv.VerifyStandaloneReady(ctx, deployment, name, standalone); err != nil { + return nil, fmt.Errorf("standalone not ready: %w", err) + } + return standalone, nil +} + +// DeployAndVerifyMonitoringConsole deploys a Monitoring Console and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyMonitoringConsole(ctx context.Context, deployment *Deployment, name string, licenseManagerRef string) (*enterpriseApi.MonitoringConsole, error) { + mc, err := deployment.DeployMonitoringConsole(ctx, name, licenseManagerRef) + if err != nil { + return nil, fmt.Errorf("unable to deploy Monitoring Console instance: %w", err) + } + if err := testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, name, mc); err != nil { + return nil, fmt.Errorf("monitoring console not ready: %w", err) + } + return mc, nil +} + +// VerifyIndexerCPULimits verifies CPU limits on all indexer pods in a single-site cluster +func (testcaseenv *TestCaseEnv) VerifyIndexerCPULimits(deployment *Deployment, indexerCount int, expectedCPULimit string) error { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deployment.GetName(), i) + if err := testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit); err != nil { + return err + } + } + return nil +} + +// VerifySearchHeadCPULimits verifies CPU limits on all search head pods +func (testcaseenv *TestCaseEnv) VerifySearchHeadCPULimits(deployment *Deployment, searchHeadCount int, expectedCPULimit string) error { + for i := 0; i < searchHeadCount; i++ { + podName := fmt.Sprintf(SearchHeadPod, deployment.GetName(), i) + if err := testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit); err != nil { + return err + } + } + return nil +} + +// VerifyM4ComponentsReady verifies multisite indexers, multisite status, and SHC are ready (without CM check or RFSF). +func (testcaseenv *TestCaseEnv) VerifyM4ComponentsReady(ctx context.Context, deployment *Deployment, siteCount int) error { + if err := testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + if err := testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount); err != nil { + return err + } + return testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func (testcaseenv *TestCaseEnv) VerifyMCVersionChangedAndReady(ctx context.Context, deployment *Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) error { + if err := testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion); err != nil { + return err + } + return testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} + +// VerifyClusterReadyAndRFSF is a V4-only verification pattern that checks C3 cluster is ready (using ClusterManager) and RF/SF is met +func (testcaseenv *TestCaseEnv) VerifyClusterReadyAndRFSF(ctx context.Context, deployment *Deployment) error { + if err := testcaseenv.VerifyC3ClusterReady(ctx, deployment, testcaseenv.VerifyClusterManagerReady); err != nil { + return err + } + return testcaseenv.VerifyRFSFMet(ctx, deployment) +} + +// TriggerAndVerifyTelemetry is a common pattern for telemetry verification +func (testcaseenv *TestCaseEnv) TriggerAndVerifyTelemetry(ctx context.Context, deployment *Deployment, prevSubmissionTime string) error { + testcaseenv.TriggerTelemetrySubmission(ctx, deployment) + return testcaseenv.VerifyTelemetry(ctx, deployment, prevSubmissionTime) +} + +// VerifyProbeConfigAndScripts verifies probe config map exists and probe scripts are present on all pods. +// If includeStartup is true, the startup probe script is also checked. +func (testcaseenv *TestCaseEnv) VerifyProbeConfigAndScripts(ctx context.Context, deployment *Deployment, includeStartup bool) error { + testcaseenv.Log.Info("Get config map for livenessProbe and readinessProbe") + configMapName := enterprise.GetProbeConfigMapName(testcaseenv.GetName()) + _, err := GetConfigMap(ctx, deployment, testcaseenv.GetName(), configMapName) + if err != nil { + return fmt.Errorf("unable to get config map for livenessProbe and readinessProbe %s: %w", configMapName, err) + } + scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} + if includeStartup { + scriptsNames = append(scriptsNames, enterprise.GetStartupScriptName()) + } + allPods := DumpGetPods(testcaseenv.GetName()) + return testcaseenv.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) +} + +// NewStandaloneSpecWithMCRef creates a StandaloneSpec with a MonitoringConsoleRef set to the given MC name. +func NewStandaloneSpecWithMCRef(image string, mcName string) enterpriseApi.StandaloneSpec { + return enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: image, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } +} + +// VerifyLMConfiguredOnMC verifies that the License Manager is configured on the Monitoring Console pod. +func VerifyLMConfiguredOnMC(ctx context.Context, deployment *Deployment) error { + monitoringConsolePodName := fmt.Sprintf(MonitoringConsolePod, deployment.GetName()) + return VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) +} + +// StandardC3Verification performs the standard V4-only set of verifications for a C3 cluster. +// This includes cluster ready (ClusterManager), RF/SF met, and monitoring console ready. +func (testcaseenv *TestCaseEnv) StandardC3Verification(ctx context.Context, deployment *Deployment, mc *enterpriseApi.MonitoringConsole) error { + if err := testcaseenv.VerifyClusterReadyAndRFSF(ctx, deployment); err != nil { + return err + } + return testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} + +// DeployAndVerifyC3 deploys a C3 single-site cluster and verifies it reaches the ready state. +func (c *ClusterReadinessConfig) DeployAndVerifyC3(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv, replicas int, shc bool, mcRef string) error { + if err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), replicas, shc, mcRef); err != nil { + return fmt.Errorf("unable to deploy C3 cluster: %w", err) + } + return c.VerifyC3ClusterReady(ctx, deployment, testcaseEnv) +} + +// DeployAndVerifyM4 deploys an M4 multisite cluster and verifies the Cluster Manager +// and all M4 components reach the ready state. +func (c *ClusterReadinessConfig) DeployAndVerifyM4(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv, indexerReplicas int, siteCount int, mcRef string) error { + if err := c.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), indexerReplicas, siteCount, mcRef); err != nil { + return fmt.Errorf("unable to deploy M4 cluster: %w", err) + } + if err := c.ClusterManagerReady(ctx, deployment, testcaseEnv); err != nil { + return err + } + return testcaseEnv.VerifyM4ComponentsReady(ctx, deployment, siteCount) +} + +// DeployC3WithLicense sets up the license config map, deploys a C3 cluster, +// and verifies both the License Manager and cluster reach the ready state. +func (c *ClusterReadinessConfig) DeployC3WithLicense(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv, replicas int, shc bool, mcRef string) error { + if err := SetupLicenseConfigMap(ctx, testcaseEnv); err != nil { + return err + } + if err := c.DeployAndVerifyC3(ctx, deployment, testcaseEnv, replicas, shc, mcRef); err != nil { + return err + } + return c.LicenseManagerReady(ctx, deployment, testcaseEnv) +} + +// DeployM4WithLicense sets up the license config map, deploys an M4 multisite cluster, +// and verifies the License Manager, Cluster Manager, and all M4 components reach the ready state. +func (c *ClusterReadinessConfig) DeployM4WithLicense(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv, indexerReplicas int, siteCount int, mcRef string) error { + if err := SetupLicenseConfigMap(ctx, testcaseEnv); err != nil { + return err + } + if err := c.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), indexerReplicas, siteCount, mcRef); err != nil { + return fmt.Errorf("unable to deploy M4 cluster: %w", err) + } + if err := c.LicenseManagerReady(ctx, deployment, testcaseEnv); err != nil { + return err + } + if err := c.ClusterManagerReady(ctx, deployment, testcaseEnv); err != nil { + return err + } + return testcaseEnv.VerifyM4ComponentsReady(ctx, deployment, siteCount) +} diff --git a/test/testenv/cr_update_helpers.go b/test/testenv/cr_update_helpers.go new file mode 100644 index 000000000..4bf46a963 --- /dev/null +++ b/test/testenv/cr_update_helpers.go @@ -0,0 +1,33 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetAndDeleteCR fetches a CR by name and deletes it, returning any error encountered. +func GetAndDeleteCR(ctx context.Context, deployment *Deployment, obj client.Object, instanceName string) error { + if err := deployment.GetInstance(ctx, instanceName, obj); err != nil { + return fmt.Errorf("unable to get instance %s: %w", instanceName, err) + } + if err := deployment.DeleteCR(ctx, obj); err != nil { + return fmt.Errorf("unable to delete instance %s: %w", instanceName, err) + } + return nil +} diff --git a/test/testenv/deployment.go b/test/testenv/deployment.go index 395722983..4bb968af0 100644 --- a/test/testenv/deployment.go +++ b/test/testenv/deployment.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -110,7 +110,7 @@ func (d *Deployment) Teardown() error { var err error var output []byte - podName := GetOperatorPodName(d.testenv) + podName := d.testenv.GetOperatorPodName() if d.testenv.clusterWideOperator != "true" { output, err = exec.Command("kubectl", "logs", "-n", d.testenv.GetName(), podName).Output() } else { @@ -154,7 +154,7 @@ func (d *Deployment) DeployStandalone(ctx context.Context, name string, mcRef st // If license file specified, deploy License Manager if LicenseManager != "" && d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return nil, err @@ -326,7 +326,7 @@ func (d *Deployment) OperatorPodExecCommand(ctx context.Context, podName string, return stdout.String(), stderr.String(), nil } -// DeployLicenseManager deploys the license manager instance +// DeployLicenseManager deploys the License Manager instance func (d *Deployment) DeployLicenseManager(ctx context.Context, name string) (*enterpriseApi.LicenseManager, error) { if d.testenv.licenseFilePath == "" { @@ -344,7 +344,7 @@ func (d *Deployment) DeployLicenseManager(ctx context.Context, name string) (*en return deployed.(*enterpriseApi.LicenseManager), err } -// DeployLicenseMaster deploys the license manager instance +// DeployLicenseMaster deploys the License Master instance func (d *Deployment) DeployLicenseMaster(ctx context.Context, name string) (*enterpriseApiV3.LicenseMaster, error) { if d.testenv.licenseFilePath == "" { @@ -362,9 +362,9 @@ func (d *Deployment) DeployLicenseMaster(ctx context.Context, name string) (*ent return deployed.(*enterpriseApiV3.LicenseMaster), err } -// DeployClusterManager deploys the cluster manager +// DeployClusterManager deploys the Cluster Manager func (d *Deployment) DeployClusterManager(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name, "LicenseRef", LicenseManagerName) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name, "LicenseRef", LicenseManagerName) cm := newClusterManager(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { cm.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -382,9 +382,9 @@ func (d *Deployment) DeployClusterManager(ctx context.Context, name, LicenseMana return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMaster deploys the cluster manager +// DeployClusterMaster deploys the Cluster Master func (d *Deployment) DeployClusterMaster(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-master", "name", name, "LicenseRef", LicenseManagerName) + d.testenv.Log.Info("Deploying Cluster Master", "name", name, "LicenseRef", LicenseManagerName) cm := newClusterMaster(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { cm.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -402,9 +402,9 @@ func (d *Deployment) DeployClusterMaster(ctx context.Context, name, LicenseManag return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeployClusterManagerWithSmartStoreIndexes deploys the cluster manager with smartstore indexes +// DeployClusterManagerWithSmartStoreIndexes deploys the Cluster Manager with SmartStore indexes func (d *Deployment) DeployClusterManagerWithSmartStoreIndexes(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, smartstorespec enterpriseApi.SmartStoreSpec) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterManagerWithGivenIndexes(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage, smartstorespec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -416,9 +416,9 @@ func (d *Deployment) DeployClusterManagerWithSmartStoreIndexes(ctx context.Conte return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMasterWithSmartStoreIndexes deploys the cluster manager with smartstore indexes +// DeployClusterMasterWithSmartStoreIndexes deploys the Cluster Master with SmartStore indexes func (d *Deployment) DeployClusterMasterWithSmartStoreIndexes(ctx context.Context, name, LicenseManagerName, ansibleConfig string, smartstorespec enterpriseApi.SmartStoreSpec) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterMasterWithGivenIndexes(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage, smartstorespec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -430,12 +430,12 @@ func (d *Deployment) DeployClusterMasterWithSmartStoreIndexes(ctx context.Contex return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeployIndexerCluster deploys the indexer cluster +// DeployIndexerCluster deploys the Indexer Cluster func (d *Deployment) DeployIndexerCluster(ctx context.Context, name, LicenseManagerName string, count int, clusterManagerRef string, ansibleConfig string, queue, os corev1.ObjectReference, serviceAccountName string) (*enterpriseApi.IndexerCluster, error) { - d.testenv.Log.Info("Deploying indexer cluster", "name", name, "CM", clusterManagerRef) + d.testenv.Log.Info("Deploying Indexer Cluster", "name", name, "CM", clusterManagerRef) indexer := newIndexerCluster(name, d.testenv.namespace, LicenseManagerName, count, clusterManagerRef, ansibleConfig, d.testenv.splunkImage, queue, os, serviceAccountName) pdata, _ := json.Marshal(indexer) - d.testenv.Log.Info("indexer cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Indexer Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, name, indexer) if err != nil { return nil, err @@ -444,14 +444,14 @@ func (d *Deployment) DeployIndexerCluster(ctx context.Context, name, LicenseMana return deployed.(*enterpriseApi.IndexerCluster), err } -// DeployIngestorCluster deploys the ingestor cluster +// DeployIngestorCluster deploys the Ingestor Cluster func (d *Deployment) DeployIngestorCluster(ctx context.Context, name string, count int, queue, os corev1.ObjectReference, serviceAccountName string) (*enterpriseApi.IngestorCluster, error) { - d.testenv.Log.Info("Deploying ingestor cluster", "name", name) + d.testenv.Log.Info("Deploying Ingestor Cluster", "name", name) ingestor := newIngestorCluster(name, d.testenv.namespace, count, d.testenv.splunkImage, queue, os, serviceAccountName) pdata, _ := json.Marshal(ingestor) - d.testenv.Log.Info("ingestor cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Ingestor Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, name, ingestor) if err != nil { return nil, err @@ -492,13 +492,13 @@ func (d *Deployment) DeployObjectStorage(ctx context.Context, name string, objSt return deployed.(*enterpriseApi.ObjectStorage), err } -// DeployIngestorClusterWithAdditionalConfiguration deploys the ingestor cluster with additional configuration +// DeployIngestorClusterWithAdditionalConfiguration deploys the Ingestor Cluster with additional configuration func (d *Deployment) DeployIngestorClusterWithAdditionalConfiguration(ctx context.Context, ic *enterpriseApi.IngestorCluster) (*enterpriseApi.IngestorCluster, error) { - d.testenv.Log.Info("Deploying ingestor cluster with additional configuration", "name", ic.Name) + d.testenv.Log.Info("Deploying Ingestor Cluster with additional configuration", "name", ic.Name) pdata, _ := json.Marshal(ic) - d.testenv.Log.Info("ingestor cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Ingestor Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, ic.Name, ic) if err != nil { return nil, err @@ -507,9 +507,9 @@ func (d *Deployment) DeployIngestorClusterWithAdditionalConfiguration(ctx contex return deployed.(*enterpriseApi.IngestorCluster), err } -// DeploySearchHeadCluster deploys a search head cluster +// DeploySearchHeadCluster deploys a Search Head Cluster func (d *Deployment) DeploySearchHeadCluster(ctx context.Context, name, ClusterManagerRef, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApi.SearchHeadCluster, error) { - d.testenv.Log.Info("Deploying search head cluster", "name", name) + d.testenv.Log.Info("Deploying Search Head Cluster", "name", name) sh := newSearchHeadCluster(name, d.testenv.namespace, ClusterManagerRef, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { sh.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -553,7 +553,7 @@ func (d *Deployment) deployCR(ctx context.Context, name string, cr client.Object return nil }) - // Returns once we can retrieve the lm instance + // Returns once we can retrieve the LM instance if err := wait.PollImmediate(PollInterval, DefaultTimeout, func() (bool, error) { key := client.ObjectKey{Name: name, Namespace: d.testenv.namespace} err := d.testenv.GetKubeClient().Get(ctx, key, cr) @@ -725,7 +725,7 @@ func (d *Deployment) DeleteCR(ctx context.Context, cr client.Object) error { return err } -// DeploySingleSiteCluster deploys a lm and indexer cluster (shc optional) +// DeploySingleSiteCluster deploys a LM and Indexer Cluster (SHC optional) func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, indexerReplicas int, shc bool, mcRef string) error { var LicenseManager string @@ -734,13 +734,13 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i if d.testenv.licenseFilePath != "" { // Enable LM to be tested if strings.Contains(name, "master") { - // Deploy the license master + // Deploy the License Master _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err } } else { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -751,26 +751,26 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i } if strings.Contains(name, "master") { - // Deploy the cluster master + // Deploy the Cluster Master _, err := d.DeployClusterMaster(ctx, name, LicenseManager, "", mcRef) if err != nil { return err } } else { - // Deploy the cluster manager + // Deploy the Cluster Manager _, err := d.DeployClusterManager(ctx, name, LicenseManager, "", mcRef) if err != nil { return err } } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err := d.DeployIndexerCluster(ctx, name+"-idxc", LicenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err } - // Deploy the SH cluster + // Deploy the SH Cluster if shc { _, err = d.DeploySearchHeadCluster(ctx, name+"-shc", name, LicenseManager, "", mcRef) if err != nil { @@ -781,7 +781,7 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i return nil } -// DeployMultisiteClusterMasterWithSearchHead deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterMasterWithSearchHead deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseMaster string @@ -796,7 +796,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Cont LicenseMaster = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -843,7 +843,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Cont return nil } -// DeployMultisiteClusterWithSearchHead deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterWithSearchHead deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseManager string @@ -852,13 +852,13 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n if d.testenv.licenseFilePath != "" { // Enable LM to be tested if strings.Contains(name, "master") { - // Deploy the license master + // Deploy the License Master _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err } } else { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -868,7 +868,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -915,14 +915,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n return nil } -// DeployMultisiteCluster deploys a lm, cluster-manager, indexers in multiple sites +// DeployMultisiteCluster deploys a LM, Cluster Manager, and indexers in multiple sites func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -931,7 +931,7 @@ func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, in LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -965,13 +965,13 @@ func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, in return nil } -// DeployStandaloneWithLMaster deploys a standalone splunk enterprise instance with license manager on the specified testenv +// DeployStandaloneWithLMaster deploys a Standalone Splunk Enterprise instance with License Master on the specified testenv func (d *Deployment) DeployStandaloneWithLMaster(ctx context.Context, name string, mcRef string) (*enterpriseApi.Standalone, error) { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return nil, err @@ -992,13 +992,13 @@ func (d *Deployment) DeployStandaloneWithLMaster(ctx context.Context, name strin return deployed.(*enterpriseApi.Standalone), err } -// DeployStandaloneWithLM deploys a standalone splunk enterprise instance with license manager on the specified testenv +// DeployStandaloneWithLM deploys a Standalone Splunk Enterprise instance with License Manager on the specified testenv func (d *Deployment) DeployStandaloneWithLM(ctx context.Context, name string, mcRef string) (*enterpriseApi.Standalone, error) { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return nil, err @@ -1051,14 +1051,14 @@ func (d *Deployment) DeployStandaloneWithGivenSmartStoreSpec(ctx context.Context return deployed.(*enterpriseApi.Standalone), err } -// DeployMultisiteClusterWithSearchHeadAndIndexes deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterWithSearchHeadAndIndexes deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context.Context, name string, indexerReplicas int, siteCount int, indexesSecret string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1067,7 +1067,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context. LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1106,14 +1106,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context. return err } -// DeployMultisiteClusterMasterWithSearchHeadAndIndexes deploys a lm, cluster-master, indexers in multiple sites and SH clusters +// DeployMultisiteClusterMasterWithSearchHeadAndIndexes deploys a LM, Cluster Master, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx context.Context, name string, indexerReplicas int, siteCount int, indexesSecret string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1122,7 +1122,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx co LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1161,9 +1161,9 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx co return err } -// DeployClusterManagerWithGivenSpec deploys the cluster manager with given SPEC +// DeployClusterManagerWithGivenSpec deploys the Cluster Manager with given SPEC func (d *Deployment) DeployClusterManagerWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.ClusterManagerSpec) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterManagerWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -1172,9 +1172,9 @@ func (d *Deployment) DeployClusterManagerWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMasterWithGivenSpec deploys the cluster manager with given SPEC +// DeployClusterMasterWithGivenSpec deploys the Cluster Master with given SPEC func (d *Deployment) DeployClusterMasterWithGivenSpec(ctx context.Context, name string, spec enterpriseApiV3.ClusterMasterSpec) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-master", "name", name) + d.testenv.Log.Info("Deploying Cluster Master", "name", name) cm := newClusterMasterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -1183,17 +1183,17 @@ func (d *Deployment) DeployClusterMasterWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeploySearchHeadClusterWithGivenSpec deploys a search head cluster +// DeploySearchHeadClusterWithGivenSpec deploys a Search Head Cluster func (d *Deployment) DeploySearchHeadClusterWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.SearchHeadClusterSpec) (*enterpriseApi.SearchHeadCluster, error) { - d.testenv.Log.Info("Deploying search head cluster", "name", name) + d.testenv.Log.Info("Deploying Search Head Cluster", "name", name) indexer := newSearchHeadClusterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, indexer) return deployed.(*enterpriseApi.SearchHeadCluster), err } -// DeployLicenseManagerWithGivenSpec deploys the license manager with given SPEC +// DeployLicenseManagerWithGivenSpec deploys the License Manager with given SPEC func (d *Deployment) DeployLicenseManagerWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.LicenseManagerSpec) (*enterpriseApi.LicenseManager, error) { - d.testenv.Log.Info("Deploying license-manager", "name", name) + d.testenv.Log.Info("Deploying License Manager", "name", name) lm := newLicenseManagerWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, lm) if err != nil { @@ -1202,9 +1202,9 @@ func (d *Deployment) DeployLicenseManagerWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApi.LicenseManager), err } -// DeployLicenseMasterWithGivenSpec deploys the license manager with given SPEC +// DeployLicenseMasterWithGivenSpec deploys the License Master with given SPEC func (d *Deployment) DeployLicenseMasterWithGivenSpec(ctx context.Context, name string, spec enterpriseApiV3.LicenseMasterSpec) (*enterpriseApiV3.LicenseMaster, error) { - d.testenv.Log.Info("Deploying license-master", "name", name) + d.testenv.Log.Info("Deploying License Master", "name", name) lm := newLicenseMasterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, lm) if err != nil { @@ -1213,7 +1213,7 @@ func (d *Deployment) DeployLicenseMasterWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApiV3.LicenseMaster), err } -// DeploySingleSiteClusterWithGivenAppFrameworkSpec deploys indexer cluster (lm, shc optional) with app framework spec +// DeploySingleSiteClusterWithGivenAppFrameworkSpec deploys Indexer Cluster (LM, SHC optional) with app framework spec func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx context.Context, name string, indexerReplicas int, shc bool, appFrameworkSpecIdxc enterpriseApi.AppFrameworkSpec, appFrameworkSpecShc enterpriseApi.AppFrameworkSpec, mcName string, licenseManager string) (*enterpriseApi.ClusterManager, *enterpriseApi.IndexerCluster, *enterpriseApi.SearchHeadCluster, error) { cm := &enterpriseApi.ClusterManager{} @@ -1222,14 +1222,14 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster manager + // Deploy the Cluster Manager cmSpec := enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1251,7 +1251,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex return cm, idxc, sh, err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster idxc, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return cm, idxc, sh, err @@ -1279,7 +1279,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex } pdata, _ := json.Marshal(shSpec) - d.testenv.Log.Info("Search head Spec", "cr", pdata) + d.testenv.Log.Info("Search Head Spec", "cr", pdata) if shc { sh, err = d.DeploySearchHeadClusterWithGivenSpec(ctx, name+"-shc", shSpec) @@ -1291,7 +1291,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex return cm, idxc, sh, nil } -// DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec deploys indexer cluster (lm, shc optional) with app framework spec +// DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec deploys Indexer Cluster (LM, SHC optional) with app framework spec func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx context.Context, name string, indexerReplicas int, shc bool, appFrameworkSpecIdxc enterpriseApi.AppFrameworkSpec, appFrameworkSpecShc enterpriseApi.AppFrameworkSpec, mcName string, licenseMaster string) (*enterpriseApiV3.ClusterMaster, *enterpriseApi.IndexerCluster, *enterpriseApi.SearchHeadCluster, error) { cm := &enterpriseApiV3.ClusterMaster{} @@ -1300,14 +1300,14 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster manager + // Deploy the Cluster Master cmSpec := enterpriseApiV3.ClusterMasterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1329,7 +1329,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx return cm, idxc, sh, err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster idxc, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseMaster, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return cm, idxc, sh, err @@ -1357,7 +1357,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx } pdata, _ := json.Marshal(shSpec) - d.testenv.Log.Info("Search head Spec", "cr", pdata) + d.testenv.Log.Info("Search Head Spec", "cr", pdata) if shc { sh, err = d.DeploySearchHeadClusterWithGivenSpec(ctx, name+"-shc", shSpec) @@ -1378,14 +1378,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx con // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, licenseManager) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1440,7 +1440,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx con multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-manager") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1482,14 +1482,14 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, licenseMaster) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1544,7 +1544,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-master") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1577,14 +1577,14 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c return cm, idxc, sh, nil } -// DeploySingleSiteClusterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console +// DeploySingleSiteClusterWithGivenMonitoringConsole deploys Indexer Cluster (LM, SHC optional) with given Monitoring Console func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx context.Context, name string, indexerReplicas int, shc bool, monitoringConsoleName string) error { licenseManager := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1593,7 +1593,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte licenseManager = name } - // Deploy the cluster manager + // Deploy the Cluster Manager cmSpec := enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1614,7 +1614,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte return err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err @@ -1649,14 +1649,14 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte return nil } -// DeploySingleSiteClusterMasterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console +// DeploySingleSiteClusterMasterWithGivenMonitoringConsole deploys Indexer Cluster (LM, SHC optional) with given Monitoring Console func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx context.Context, name string, indexerReplicas int, shc bool, monitoringConsoleName string) error { licenseMaster := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1665,7 +1665,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx licenseMaster = name } - // Deploy the cluster manager + // Deploy the Cluster Master cmSpec := enterpriseApiV3.ClusterMasterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1686,7 +1686,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx return err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseMaster, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err @@ -1728,7 +1728,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1737,7 +1737,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con licenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1791,7 +1791,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-manager") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1828,7 +1828,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1837,7 +1837,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte licenseMaster = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1891,7 +1891,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-master") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1930,3 +1930,46 @@ func (d *Deployment) GetConfigMap(ctx context.Context, name string) (*corev1.Con } return cm, nil } + +// S1WithLMSetup holds the resources created by SetupS1WithLMAndMC so that +// individual test functions can operate on them without repeating the setup. +type S1WithLMSetup struct { + Standalone *enterpriseApi.Standalone + Mc *enterpriseApi.MonitoringConsole + ResourceVersion string + NamespaceScopedSecretName string +} + +// SetupS1WithLMAndMC performs the common S1 setup shared by the secret-update +// and secret-delete tests: license config map, standalone with LM, MC, and +// initial secret verification. +func SetupS1WithLMAndMC(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig) (S1WithLMSetup, error) { + if err := SetupLicenseConfigMap(ctx, testcaseEnvInst); err != nil { + return S1WithLMSetup{}, err + } + + mcRef := deployment.GetName() + standalone, err := config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) + if err != nil { + return S1WithLMSetup{}, fmt.Errorf("unable to deploy standalone instance with LM: %w", err) + } + + VerifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + if err != nil { + return S1WithLMSetup{}, fmt.Errorf("unable to deploy Monitoring Console: %w", err) + } + + namespaceScopedSecretName := fmt.Sprintf(NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + if _, err = GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName); err != nil { + return S1WithLMSetup{}, fmt.Errorf("unable to get secret struct: %w", err) + } + + return S1WithLMSetup{ + Standalone: standalone, + Mc: mc, + ResourceVersion: resourceVersion, + NamespaceScopedSecretName: namespaceScopedSecretName, + }, nil +} diff --git a/test/testenv/filespec.go b/test/testenv/filespec.go index 8e9131955..3dddb158b 100644 --- a/test/testenv/filespec.go +++ b/test/testenv/filespec.go @@ -99,11 +99,6 @@ func (p remotePath) Join(elem pathSpec) remotePath { return newRemotePath(path.Join(p.file, elem.String())) } -func (p remotePath) StripShortcuts() remotePath { - p = p.Clean() - return newRemotePath(stripPathShortcuts(p.file)) -} - func (p remotePath) StripSlashes() remotePath { return newRemotePath(stripLeadingSlash(p.file)) } @@ -123,25 +118,3 @@ func stripLeadingSlash(file string) string { // tar strips the leading '/' and '\' if it's there, so we will too return strings.TrimLeft(file, `/\`) } - -// stripPathShortcuts removes any leading or trailing "../" from a given path -func stripPathShortcuts(p string) string { - newPath := p - trimmed := strings.TrimPrefix(newPath, "../") - - for trimmed != newPath { - newPath = trimmed - trimmed = strings.TrimPrefix(newPath, "../") - } - - // trim leftover {".", ".."} - if newPath == "." || newPath == ".." { - newPath = "" - } - - if len(newPath) > 0 && string(newPath[0]) == "/" { - return newPath[1:] - } - - return newPath -} diff --git a/test/testenv/gcputils.go b/test/testenv/gcputils.go index 78c41d1aa..e6b9a0b1e 100644 --- a/test/testenv/gcputils.go +++ b/test/testenv/gcputils.go @@ -120,9 +120,9 @@ func CheckPrefixExistsOnGCP(prefix string) bool { logf.Log.Error(err, "Error listing objects in GCP bucket") return false } - logf.Log.Info("CHECKING OBJECT", "OBJECT", objAttrs.Name) + logf.Log.Info("CHECKING OBJECT", "object", objAttrs.Name) if strings.Contains(objAttrs.Name, prefix) { - logf.Log.Info("Prefix found in bucket", "Prefix", prefix, "Object", objAttrs.Name) + logf.Log.Info("Prefix found in bucket", "prefix", prefix, "object", objAttrs.Name) return true } } @@ -146,12 +146,12 @@ func CreateBucketAndPathIfNotExist(bucketName, path string) error { // Create the bucket err = client.Client.Bucket(bucketName).Create(ctx, gcpProjectID, nil) if err != nil { - logf.Log.Error(err, "Failed to create bucket", "Bucket Name", bucketName) + logf.Log.Error(err, "Failed to create bucket", "bucketName", bucketName) return err } - logf.Log.Info("Bucket created", "Bucket Name", bucketName) + logf.Log.Info("Bucket created", "bucketName", bucketName) } else if err != nil { - logf.Log.Error(err, "Error checking bucket attributes", "Bucket Name", bucketName) + logf.Log.Error(err, "Error checking bucket attributes", "bucketName", bucketName) return err } @@ -161,16 +161,16 @@ func CreateBucketAndPathIfNotExist(bucketName, path string) error { // Create a zero-length object to represent the path wc := client.Client.Bucket(bucketName).Object(path).NewWriter(ctx) if _, err := wc.Write([]byte{}); err != nil { - logf.Log.Error(err, "Failed to create path", "Path", path) + logf.Log.Error(err, "Failed to create path", "path", path) return err } if err := wc.Close(); err != nil { - logf.Log.Error(err, "Failed to finalize path creation", "Path", path) + logf.Log.Error(err, "Failed to finalize path creation", "path", path) return err } - logf.Log.Info("Path created", "Path", path) + logf.Log.Info("Path created", "path", path) } else if err != nil { - logf.Log.Error(err, "Error checking path attributes", "Path", path) + logf.Log.Error(err, "Error checking path attributes", "path", path) return err } @@ -209,7 +209,7 @@ func DownloadFileFromGCP(bucketName, objectName, gcpFilePath, downloadDir string objectPath := filepath.Join(gcpFilePath, objectName) rc, err := client.Client.Bucket(bucketName).Object(objectPath).NewReader(ctx) if err != nil { - logf.Log.Error(err, "Failed to create reader for object", "Object", objectName) + logf.Log.Error(err, "Failed to create reader for object", "object", objectName) return "", err } defer rc.Close() @@ -217,14 +217,14 @@ func DownloadFileFromGCP(bucketName, objectName, gcpFilePath, downloadDir string localPath := filepath.Join(downloadDir, objectName) file, err := os.Create(localPath) if err != nil { - logf.Log.Error(err, "Failed to create local file", "Filename", localPath) + logf.Log.Error(err, "Failed to create local file", "filename", localPath) return "", err } defer file.Close() written, err := io.Copy(file, rc) if err != nil { - logf.Log.Error(err, "Failed to download object", "Object", objectName) + logf.Log.Error(err, "Failed to download object", "object", objectName) return "", err } @@ -319,18 +319,18 @@ func DeleteFileOnGCP(bucketName, objectName string) error { err = client.Client.Bucket(bucketName).Object(objectName).Delete(ctx) if err != nil && err != storage.ErrObjectNotExist { - logf.Log.Error(err, "Unable to delete object from bucket", "Object Name", objectName, "Bucket Name", bucketName) + logf.Log.Error(err, "Unable to delete object from bucket", "objectName", objectName, "bucketName", bucketName) return err } // Optionally, verify deletion _, err = client.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) if err == storage.ErrObjectNotExist { - logf.Log.Info("Deleted file on GCP", "File Name", objectName, "Bucket", bucketName) + logf.Log.Info("Deleted file on GCP", "fileName", objectName, "bucket", bucketName) return nil } if err != nil { - logf.Log.Error(err, "Error verifying deletion of object", "Object Name", objectName, "Bucket Name", bucketName) + logf.Log.Error(err, "Error verifying deletion of object", "objectName", objectName, "bucketName", bucketName) return err } @@ -342,12 +342,12 @@ func GetFilesInPathOnGCP(bucketName, path string) []string { resp := GetFileListOnGCP(bucketName, path) var files []string for _, obj := range resp { - logf.Log.Info("CHECKING OBJECT", "OBJECT", obj.Name) + logf.Log.Info("CHECKING OBJECT", "object", obj.Name) if strings.HasPrefix(obj.Name, path) { filename := strings.TrimPrefix(obj.Name, path) // This condition filters out directories as GCP returns objects with their full paths if len(filename) > 1 && !strings.HasSuffix(filename, "/") { - logf.Log.Info("File found in bucket", "Path", path, "Object", obj.Name) + logf.Log.Info("File found in bucket", "path", path, "object", obj.Name) files = append(files, filename) } } @@ -358,10 +358,10 @@ func GetFilesInPathOnGCP(bucketName, path string) []string { // DownloadFilesFromGCP downloads a list of files from a GCP bucket to a local directory func DownloadFilesFromGCP(bucketName, gcpAppDir, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading file from GCP", "File name", key) + logf.Log.Info("Downloading file from GCP", "fileName", key) _, err := DownloadFileFromGCP(bucketName, key, gcpAppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -372,22 +372,22 @@ func DownloadFilesFromGCP(bucketName, gcpAppDir, downloadDir string, appList []s func UploadFilesToGCP(bucketName, gcpTestDir string, appList []string, uploadDir string) ([]string, error) { var uploadedFiles []string for _, key := range appList { - logf.Log.Info("Uploading file to GCP", "File name", key) - logf.Log.Info("Using bucket", "Bucket", bucketName, "Path", gcpTestDir, "Upload Dir", uploadDir) + logf.Log.Info("Uploading file to GCP", "fileName", key) + logf.Log.Info("Using bucket", "bucket", bucketName, "path", gcpTestDir, "uploadDir", uploadDir) fileLocation := filepath.Join(uploadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open file", "File name", key) + logf.Log.Error(err, "Unable to open file", "fileName", key) return nil, err } defer fileBody.Close() objectPath, err := UploadFileToGCP(bucketName, key, gcpTestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } - logf.Log.Info("File uploaded to GCP", "File name", objectPath) + logf.Log.Info("File uploaded to GCP", "fileName", objectPath) uploadedFiles = append(uploadedFiles, objectPath) } return uploadedFiles, nil diff --git a/test/testenv/index_ingest_separation_utils.go b/test/testenv/index_ingest_separation_utils.go new file mode 100644 index 000000000..dcd03019a --- /dev/null +++ b/test/testenv/index_ingest_separation_utils.go @@ -0,0 +1,91 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package testenv + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" +) + +// DeployQueueAndObjectStorage deploys a Queue and ObjectStorage CR and returns both. +func DeployQueueAndObjectStorage(ctx context.Context, deployment *Deployment, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec) (*enterpriseApi.Queue, *enterpriseApi.ObjectStorage, error) { + q, err := deployment.DeployQueue(ctx, "queue", qSpec) + if err != nil { + return nil, nil, fmt.Errorf("unable to deploy Queue: %w", err) + } + + objStorage, err := deployment.DeployObjectStorage(ctx, "os", osSpec) + if err != nil { + return nil, nil, fmt.Errorf("unable to deploy ObjectStorage: %w", err) + } + + return q, objStorage, nil +} + +// SetupIngestorStack deploys the full Queue/ObjectStorage/IngestorCluster/ClusterManager/IndexerCluster stack +// and verifies each component reaches the Ready phase. +func (testcaseEnvInst *TestCaseEnv) SetupIngestorStack(ctx context.Context, deployment *Deployment, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec, cmSpec enterpriseApi.ClusterManagerSpec) error { + volumeSpec := []enterpriseApi.SQSVolumeSpec{GenerateQueueVolumeSpec( + "queue-secret-ref-volume", + testcaseEnvInst.GetIndexIngestSepSecretName(), + )} + qSpec.SQS.VolList = volumeSpec + + q, objStorage, err := DeployQueueAndObjectStorage(ctx, deployment, qSpec, osSpec) + if err != nil { + return err + } + + if _, err := deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, ""); err != nil { + return fmt.Errorf("unable to deploy Ingestor Cluster: %w", err) + } + + if _, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec); err != nil { + return fmt.Errorf("unable to deploy Cluster Manager: %w", err) + } + + if _, err := deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, ""); err != nil { + return fmt.Errorf("unable to deploy Indexer Cluster: %w", err) + } + + testcaseEnvInst.VerifyIngestorReady(ctx, deployment) + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + return nil +} + +// DeleteIngestorStack tears down the full Queue/ObjectStorage/IngestorCluster/IndexerCluster stack. +func DeleteIngestorStack(ctx context.Context, deployment *Deployment) error { + // Delete the Indexer Cluster + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.IndexerCluster{}, deployment.GetName()+"-idxc"); err != nil { + return err + } + + // Delete the Ingestor Cluster + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.IngestorCluster{}, deployment.GetName()+"-ingest"); err != nil { + return err + } + + // Delete the Queue + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.Queue{}, "queue"); err != nil { + return err + } + + // Delete the ObjectStorage + return GetAndDeleteCR(ctx, deployment, &enterpriseApi.ObjectStorage{}, "os") +} diff --git a/test/testenv/ingest_utils.go b/test/testenv/ingest_utils.go index 2c0403b1e..2f1a7eb91 100644 --- a/test/testenv/ingest_utils.go +++ b/test/testenv/ingest_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -90,7 +90,7 @@ func CreateMockLogfile(logFile string, totalLines int) error { } // CreateAnIndexStandalone creates an index on a standalone instance using the CLI -func CreateAnIndexStandalone(ctx context.Context, indexName string, podName string, deployment *Deployment) error { +func CreateAnIndexStandalone(ctx context.Context, deployment *Deployment, indexName string, podName string) error { var addIndexCmd strings.Builder splunkBin := "/opt/splunk/bin/splunk" @@ -120,10 +120,10 @@ func CreateAnIndexStandalone(ctx context.Context, indexName string, podName stri } // IngestFileViaOneshot ingests a file into an instance using the oneshot CLI -func IngestFileViaOneshot(ctx context.Context, logFile string, indexName string, podName string, deployment *Deployment) error { +func IngestFileViaOneshot(ctx context.Context, deployment *Deployment, logFile string, indexName string, podName string) error { // Send it to the instance - resp, stderr, cpErr := CopyFileToPod(ctx, podName, logFile, logFile, deployment) + resp, stderr, cpErr := CopyFileToPod(ctx, deployment, podName, logFile, logFile) if cpErr != nil { logf.Log.Error(cpErr, "Failed File Copy to pod", "logFile", logFile, "podName", podName, "stderr", stderr) return cpErr @@ -158,7 +158,7 @@ func IngestFileViaOneshot(ctx context.Context, logFile string, indexName string, } // CopyFileToPod copies a file locally from srcPath to the destPath on the pod specified in podName -func CopyFileToPod(ctx context.Context, podName string, srcPath string, destPath string, deployment *Deployment) (string, string, error) { +func CopyFileToPod(ctx context.Context, deployment *Deployment, podName string, srcPath string, destPath string) (string, string, error) { // Create tar file stream reader, writer := io.Pipe() if destPath != "/" && strings.HasSuffix(string(destPath[len(destPath)-1]), "/") { @@ -224,10 +224,10 @@ func CopyFileToPod(ctx context.Context, podName string, srcPath string, destPath } // IngestFileViaMonitor ingests a file into an instance using the monitor CLI -func IngestFileViaMonitor(ctx context.Context, logFile string, indexName string, podName string, deployment *Deployment) error { +func IngestFileViaMonitor(ctx context.Context, deployment *Deployment, logFile string, indexName string, podName string) error { // Send it to the instance - resp, stderr, cpErr := CopyFileToPod(ctx, podName, logFile, logFile, deployment) + resp, stderr, cpErr := CopyFileToPod(ctx, deployment, podName, logFile, logFile) if cpErr != nil { logf.Log.Error(cpErr, "Failed File Copy to pod", "logFile", logFile, "podName", podName, "stderr", stderr) return cpErr diff --git a/test/testenv/lmutil.go b/test/testenv/lmutil.go index 9f7aa5dd7..316542cbe 100644 --- a/test/testenv/lmutil.go +++ b/test/testenv/lmutil.go @@ -1,27 +1,29 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package testenv import ( "context" "encoding/json" + "fmt" "strings" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + corev1 "k8s.io/api/core/v1" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -56,6 +58,127 @@ func CheckLicenseManagerConfigured(ctx context.Context, deployment *Deployment, return false } licenseManager := restResponse.Entry[0].Content.ManagerURI - logf.Log.Info("License Manager configuration on POD", "POD", podName, "License Manager", licenseManager) + logf.Log.Info("License Manager configuration on POD", "pod", podName, "licenseManager", licenseManager) return strings.Contains(licenseManager, "license-manager-service:8089") || strings.Contains(licenseManager, "license-master-service:8089") } + +// LicenseTestConfig holds the version-specific (V3/V4) deployment and verification +// callbacks used by the license manager test functions. +type LicenseTestConfig struct { + *ClusterReadinessConfig + DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *Deployment, name string, spec interface{}) (interface{}, error) + BuildLMAppFrameworkSpec func(testcaseEnvInst *TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} + LicenseManagerPodName string + CrKind string +} + +// NewLicenseCommonSplunkSpec returns a CommonSplunkSpec pre-configured with the +// license config map volume and license URL. +func NewLicenseCommonSplunkSpec(testcaseEnvInst *TestCaseEnv) enterpriseApi.CommonSplunkSpec { + return enterpriseApi.CommonSplunkSpec{ + Volumes: []corev1.Volume{{ + Name: "licenses", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: testcaseEnvInst.GetLMConfigMap(), + }, + }, + }, + }}, + LicenseURL: "/mnt/licenses/enterprise.lic", + Spec: enterpriseApi.Spec{ImagePullPolicy: "Always", Image: testcaseEnvInst.GetSplunkImage()}, + } +} + +// NewLicenseMasterConfig returns a LicenseTestConfig for V3 (LicenseMaster) tests. +func NewLicenseMasterConfig() *LicenseTestConfig { + return &LicenseTestConfig{ + ClusterReadinessConfig: NewClusterReadinessConfigV3(), + DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *Deployment, name string, spec interface{}) (interface{}, error) { + return deployment.DeployLicenseMasterWithGivenSpec(ctx, name, spec.(enterpriseApiV3.LicenseMasterSpec)) + }, + BuildLMAppFrameworkSpec: func(testcaseEnvInst *TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApiV3.LicenseMasterSpec{ + CommonSplunkSpec: NewLicenseCommonSplunkSpec(testcaseEnvInst), + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: LicenseMasterPod, + CrKind: "LicenseMaster", + } +} + +// NewLicenseManagerConfig returns a LicenseTestConfig for V4 (LicenseManager) tests. +func NewLicenseManagerConfig() *LicenseTestConfig { + return &LicenseTestConfig{ + ClusterReadinessConfig: NewClusterReadinessConfigV4(), + DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *Deployment, name string, spec interface{}) (interface{}, error) { + return deployment.DeployLicenseManagerWithGivenSpec(ctx, name, spec.(enterpriseApi.LicenseManagerSpec)) + }, + BuildLMAppFrameworkSpec: func(testcaseEnvInst *TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApi.LicenseManagerSpec{ + CommonSplunkSpec: NewLicenseCommonSplunkSpec(testcaseEnvInst), + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: LicenseManagerPod, + CrKind: "LicenseManager", + } +} + +// DownloadAppFiles downloads app files from the appropriate cloud provider. +func DownloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) error { + var err error + + switch ClusterProvider { + case "eks": + err = DownloadFilesFromS3(testDataS3Bucket, appDir, downloadDir, appFileList) + case "azure": + containerName := "/" + azureDataContainer + "/" + appDir + err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDir, containerName, appFileList) + case "gcp": + err = DownloadFilesFromGCP(testDataS3Bucket, appDir, downloadDir, appFileList) + } + + if err != nil { + return fmt.Errorf("unable to download %s app files: %w", version, err) + } + return nil +} + +// UploadAppFiles uploads app files to the appropriate cloud provider and returns the uploaded file paths. +func UploadAppFiles(ctx context.Context, testcaseEnvInst *TestCaseEnv, testS3Bucket, testDir, downloadDir string, appFileList []string, version string) ([]string, error) { + var uploadedFiles []string + var err error + + switch ClusterProvider { + case "eks": + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", version)) + uploadedFiles, err = UploadFilesToS3(testS3Bucket, testDir, appFileList, downloadDir) + case "azure": + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", version)) + uploadedFiles, err = UploadFilesToAzure(ctx, StorageAccount, StorageAccountKey, downloadDir, testDir, appFileList) + case "gcp": + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP", version)) + uploadedFiles, err = UploadFilesToGCP(testS3Bucket, testDir, appFileList, downloadDir) + } + + if err != nil { + return nil, fmt.Errorf("unable to upload %s apps: %w", version, err) + } + return uploadedFiles, nil +} + +// DeleteUploadedFiles removes previously uploaded app files from the appropriate cloud provider. +func DeleteUploadedFiles(ctx context.Context, testS3Bucket string, uploadedApps []string) { + switch ClusterProvider { + case "eks": + DeleteFilesOnS3(testS3Bucket, uploadedApps) + case "azure": + azureBlobClient := &AzureBlobClient{} + azureBlobClient.DeleteFilesOnAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, uploadedApps) + case "gcp": + DeleteFilesOnGCP(testS3Bucket, uploadedApps) + } +} diff --git a/test/testenv/mcutil.go b/test/testenv/mcutil.go index 36250ae96..99a7ed361 100644 --- a/test/testenv/mcutil.go +++ b/test/testenv/mcutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,7 @@ import ( "fmt" "os/exec" "strings" + "time" "github.com/splunk/splunk-operator/pkg/splunk/enterprise" corev1 "k8s.io/api/core/v1" @@ -52,30 +53,7 @@ func getMCSts(ns string) string { return strings.Split(string(output), "\n")[1] } -// CheckMCPodReady check if monitoring pod is ready. Checking status of MC pod and Stateful set. -func CheckMCPodReady(ns string) bool { - // Check Status of monitoring console statefulset - stsLine := getMCSts(ns) - if len(stsLine) == 0 { - return false - } - stsSlice := strings.Fields(stsLine) - logf.Log.Info("MC statefulset found", "POD", stsSlice[0], "READY", stsSlice[1]) - stsReady := strings.Contains(stsSlice[1], "1/1") - - // Check Status of monitoring console pod - podLine := getMCPod(ns) - if len(podLine) == 0 { - return false - } - podSlice := strings.Fields(podLine) - logf.Log.Info("MC Pod Found", "POD", podSlice[0], "READY", podSlice[1]) - podReady := strings.Contains(podSlice[1], "1/1") && strings.Contains(podSlice[2], "Running") - - return stsReady && podReady -} - -// GetConfiguredPeers get list of Peers Configured on Montioring Console +// GetConfiguredPeers get list of Peers Configured on Monitoring Console func GetConfiguredPeers(ns string, mcName string) []string { podName := fmt.Sprintf(MonitoringConsolePod, mcName) var peerList []string @@ -100,22 +78,10 @@ func GetConfiguredPeers(ns string, mcName string) []string { } } } - logf.Log.Info("Peer List found on MC Pod", "MC POD", podName, "Configured Peers", peerList) + logf.Log.Info("Peer List found on MC Pod", "mcPod", podName, "configuredPeers", peerList) return peerList } -// DeleteMCPod delete monitoring console deployment -func DeleteMCPod(ns string) { - mcSts := fmt.Sprintf(MonitoringConsoleSts, ns) - output, err := exec.Command("kubectl", "delete", "sts", "-n", ns, mcSts).Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete sts -n %s %s", ns, mcSts) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("Monitoring Console Stateful Set deleted", "Statefulset", mcSts, "stdout", output) - } -} - // CheckPodNameOnMC Check given pod is configured on Monitoring console pod func CheckPodNameOnMC(ns string, mcName string, podName string) bool { // Get Peers configured on Monitoring Console @@ -124,7 +90,7 @@ func CheckPodNameOnMC(ns string, mcName string, podName string) bool { found := false for _, peer := range peerList { if strings.Contains(peer, podName) { - logf.Log.Info("Check Peer matches on pod", "Pod Name", podName, "Peer in peer list", peer) + logf.Log.Info("Check Peer matches on pod", "podName", podName, "peerInPeerList", peer) found = true break } @@ -157,12 +123,59 @@ func GetMCConfigMap(ctx context.Context, deployment *Deployment, ns string, mcNa logf.Log.Error(err, "Failed to get Monitoring Console Config Map") return mcConfigMap, err } - logf.Log.Info("MC Config Map contents", "MC CONFIG MAP NAME", mcConfigMapName, "Data", mcConfigMap.Data) + logf.Log.Info("MC Config Map contents", "mcConfigMapName", mcConfigMapName, "data", mcConfigMap.Data) return mcConfigMap, err } // CheckPodNameInString checks for pod name in string func CheckPodNameInString(podName string, configString string) bool { - logf.Log.Info("Check MC Config String has Pod configured", "Monitoring Console Config Map Pod Config String", configString, "POD String", podName) + logf.Log.Info("Check MC Config String has Pod configured", "configString", configString, "podName", podName) return strings.Contains(configString, podName) } + +// MCReconfigParams holds the service name and URL parameters that differ between +// V3 (master) and V4 (manager) Monitoring Console tests. +type MCReconfigParams struct { + CMServiceNameFmt string // format string for CM service name (e.g., ClusterMasterServiceName) + CMURLKey string // config map URL key (e.g., "SPLUNK_CLUSTER_MASTER_URL" or splcommon.ClusterManagerURL) +} + +// MCVersionConfig captures the API-version-specific behaviour that differs +// between V3 (master) and V4 (manager) monitoring console tests. +type MCVersionConfig struct { + MCReconfigParams + + NamePrefix string + Label string + + // DeployC3WithMC deploys a C3 single-site cluster with the given MC ref. + DeployC3WithMC func(ctx context.Context, d *Deployment, name string, replicas int, shc bool, mcRef string) error + + // DeployM4WithMC deploys an M4 multisite cluster with the given MC ref. + DeployM4WithMC func(ctx context.Context, d *Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error + + // NewCMObject returns a new, empty cluster-coordinator CR + // (*ClusterMaster for V3, *ClusterManager for V4). + NewCMObject func() interface{} + + // VerifyCMReady asserts the cluster coordinator has reached Ready phase. + VerifyCMReady func(ctx context.Context, d *Deployment, te *TestCaseEnv) error + + // SHCReconfigTimeout is the timeout used when verifying MC config strings + // after an SHC MC-ref reconfig (0 means use the synchronous check). + SHCReconfigTimeout time.Duration + + // VerifyMCTwoReadyAfterSHC controls whether MC Two is explicitly + // verified ready after the SHC reconfig step. + VerifyMCTwoReadyAfterSHC bool +} + +// DeployMCAndVerifyRFSF deploys a Monitoring Console and verifies RF/SF is met. +func DeployMCAndVerifyRFSF(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, mcRef string) error { + _, err := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) + if err != nil { + return err + } + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + return nil +} diff --git a/test/testenv/multisite_helpers.go b/test/testenv/multisite_helpers.go new file mode 100644 index 000000000..bd762e819 --- /dev/null +++ b/test/testenv/multisite_helpers.go @@ -0,0 +1,89 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" +) + +// VerifyIndexOnAllSites verifies that an index exists on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnAllSites(ctx context.Context, deployment *Deployment, siteCount int, indexName string) error { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + if err := testcaseenv.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName); err != nil { + return err + } + } + return nil +} + +// IngestDataOnAllSites ingests data to an index on all indexer pods across all sites +func IngestDataOnAllSites(ctx context.Context, deployment *Deployment, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, deployment, logFile, indexName, podName) + } +} + +// RollHotToWarmOnAllSites rolls hot buckets to warm on all indexer pods across all sites +func RollHotToWarmOnAllSites(ctx context.Context, deployment *Deployment, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + RollHotToWarm(ctx, deployment, podName, indexName) + } +} + +// VerifyIndexOnS3AllSites verifies that an index exists on S3 for all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnS3AllSites(ctx context.Context, deployment *Deployment, siteCount int, indexName string) error { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + if err := testcaseenv.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName); err != nil { + return err + } + } + return nil +} + +// VerifyCPULimitsOnAllSites verifies CPU limits on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyCPULimitsOnAllSites(deployment *Deployment, siteCount int, expectedCPULimit string) error { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + if err := testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit); err != nil { + return err + } + } + return nil +} + +// MultisiteIndexerWorkflow encapsulates the common workflow for multisite indexer operations: +// verify index, ingest data, roll to warm, verify on S3 +func (testcaseenv *TestCaseEnv) MultisiteIndexerWorkflow(ctx context.Context, deployment *Deployment, siteCount int, indexName string) error { + // Verify index exists on all sites + if err := testcaseenv.VerifyIndexOnAllSites(ctx, deployment, siteCount, indexName); err != nil { + return err + } + + // Ingest data on all sites + IngestDataOnAllSites(ctx, deployment, siteCount, indexName) + + // Roll hot to warm on all sites + RollHotToWarmOnAllSites(ctx, deployment, siteCount, indexName) + + // Verify index on S3 for all sites + return testcaseenv.VerifyIndexOnS3AllSites(ctx, deployment, siteCount, indexName) +} diff --git a/test/testenv/remote_index_utils.go b/test/testenv/remote_index_utils.go index 551a377d0..a557324e4 100644 --- a/test/testenv/remote_index_utils.go +++ b/test/testenv/remote_index_utils.go @@ -3,6 +3,7 @@ package testenv import ( "context" "encoding/json" + "os" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -60,19 +61,6 @@ func GetIndexOnPod(ctx context.Context, deployment *Deployment, podName string, return indexFound, indexData } -// RestartSplunk Restart splunk inside the container -func RestartSplunk(ctx context.Context, deployment *Deployment, podName string) bool { - stdin := "/opt/splunk/bin/splunk restart -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - return true -} - // RollHotToWarm rolls hot buckets to warm for a given index and pod func RollHotToWarm(ctx context.Context, deployment *Deployment, podName string, indexName string) bool { stdin := "/opt/splunk/bin/splunk _internal call /data/indexes/" + indexName + "/roll-hot-buckets admin:$(cat /mnt/splunk-secrets/password)" @@ -130,6 +118,27 @@ func GenerateIndexVolumeSpecAzureManagedID(volumeName string, endpoint string, p } } +// GenerateVolumeSpecForProvider returns a VolumeSpec slice appropriate for the +// current ClusterProvider (eks, azure, gcp). For Azure it respects the +// AZURE_MANAGED_ID_ENABLED environment variable. +func (testenvInstance *TestCaseEnv) GenerateVolumeSpecForProvider(ctx context.Context, volumeName string) []enterpriseApi.VolumeSpec { + secretName := testenvInstance.GetIndexSecretName() + switch ClusterProvider { + case "eks": + return []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetS3Endpoint(), secretName, "aws", "s3", GetDefaultS3Region())} + case "azure": + if os.Getenv("AZURE_MANAGED_ID_ENABLED") == "false" { + return []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzure(volumeName, GetAzureEndpoint(ctx), secretName, "azure", "blob")} + } + return []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob")} + case "gcp": + return []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), secretName, "gcp", "gcs", GetDefaultS3Region())} + default: + testenvInstance.Log.Info("Failed to identify provider: Should be 'eks' or 'azure' or 'gcp'") + return nil + } +} + // GenerateIndexSpec return VolumeSpec struct with given values func GenerateIndexSpec(indexName string, volName string) enterpriseApi.IndexSpec { return enterpriseApi.IndexSpec{ diff --git a/test/testenv/s3utils.go b/test/testenv/s3utils.go index 445e395f1..3921844e2 100644 --- a/test/testenv/s3utils.go +++ b/test/testenv/s3utils.go @@ -32,11 +32,6 @@ var ( s3deleteWaitTime = 60 ) -// GetSmartStoreIndexesBucet returns smartstore test bucket name -func GetSmartStoreIndexesBucet() string { - return testIndexesS3Bucket -} - // GetDefaultS3Region returns default AWS Region func GetDefaultS3Region() string { return s3Region @@ -53,9 +48,9 @@ func CheckPrefixExistsOnS3(prefix string) bool { resp := GetFileListOnS3(dataBucket, prefix) for _, key := range resp { - logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) + logf.Log.Info("CHECKING KEY ", "key", *key.Key) if strings.Contains(*key.Key, prefix) { - logf.Log.Info("Prefix found on bucket", "Prefix", prefix, "KEY", *key.Key) + logf.Log.Info("Prefix found on bucket", "prefix", prefix, "key", *key.Key) return true } } @@ -94,7 +89,7 @@ func DownloadFileFromS3(dataBucket string, filename string, s3FilePath string, d // Create empty file on OS File System file, err := os.Create(filepath.Join(downloadDir, filename)) if err != nil { - logf.Log.Error(err, "Failed to create file", "Filename", file) + logf.Log.Error(err, "Failed to create file", "filename", file) } defer file.Close() @@ -112,7 +107,7 @@ func DownloadFileFromS3(dataBucket string, filename string, s3FilePath string, d }) if err != nil { - logf.Log.Error(err, "Failed to download file", "Bucket", dataBucket, "Path", s3FilePath, "Filename", filename) + logf.Log.Error(err, "Failed to download file", "bucket", dataBucket, "path", s3FilePath, "filename", filename) return "", err } @@ -174,7 +169,7 @@ func DeleteFileOnS3(bucket string, filename string) error { svc := s3.NewFromConfig(*cfg) _, err = svc.DeleteObject(context.TODO(), &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(filename)}) if err != nil { - logf.Log.Error(err, "Unable to delete object from bucket", "Object Name", filename, "Bucket Name", bucket) + logf.Log.Error(err, "Unable to delete object from bucket", "objectName", filename, "bucketName", bucket) } waiter := s3.NewObjectNotExistsWaiter(svc) @@ -182,7 +177,7 @@ func DeleteFileOnS3(bucket string, filename string) error { Bucket: aws.String(bucket), Key: aws.String(filename), }, time.Duration(time.Duration(s3deleteWaitTime).Seconds())) - logf.Log.Info("Deleted file on S3", "File Name", filename, "Bucket", bucket) + logf.Log.Info("Deleted file on S3", "fileName", filename, "bucket", bucket) return err } @@ -191,12 +186,12 @@ func GetFilesInPathOnS3(bucket string, path string) []string { resp := GetFileListOnS3(bucket, path) var files []string for _, key := range resp { - logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) + logf.Log.Info("CHECKING KEY ", "key", *key.Key) if strings.Contains(*key.Key, path) { filename := strings.Replace(*key.Key, path, "", -1) // This condition filters out directories as GetFileListOnS3 returns files and directories with their absolute path's if len(filename) > 1 { - logf.Log.Info("File found on bucket", "Prefix", path, "KEY", *key.Key) + logf.Log.Info("File found on bucket", "prefix", path, "key", *key.Key) files = append(files, filename) } } @@ -207,10 +202,10 @@ func GetFilesInPathOnS3(bucket string, path string) []string { // DownloadFilesFromS3 download given list of files from S3 to the given directory func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading file from S3", "File name", key) + logf.Log.Info("Downloading file from S3", "fileName", key) _, err := DownloadFileFromS3(testDataS3Bucket, key, s3AppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -221,19 +216,19 @@ func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir s func UploadFilesToS3(testS3Bucket string, s3TestDir string, applist []string, downloadDir string) ([]string, error) { var uploadedFiles []string for _, key := range applist { - logf.Log.Info("Uploading file to S3", "File name", key) + logf.Log.Info("Uploading file to S3", "fileName", key) fileLocation := filepath.Join(downloadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open file", "File name", key) + logf.Log.Error(err, "Unable to open file", "fileName", key) return nil, err } fileName, err := UploadFileToS3(testS3Bucket, key, s3TestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } - logf.Log.Info("File upload to test S3", "File name", fileName) + logf.Log.Info("File upload to test S3", "fileName", fileName) uploadedFiles = append(uploadedFiles, fileName) } return uploadedFiles, nil diff --git a/test/testenv/search_head_cluster_utils.go b/test/testenv/search_head_cluster_utils.go index 012064501..4dd326676 100644 --- a/test/testenv/search_head_cluster_utils.go +++ b/test/testenv/search_head_cluster_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,39 +18,11 @@ package testenv import ( "context" "fmt" - "os/exec" "strings" logf "sigs.k8s.io/controller-runtime/pkg/log" ) -// DeleteSHC delete Search Head Cluster in given namespace -func DeleteSHC(ns string) { - output, err := exec.Command("kubectl", "delete", "shc", "-n", ns, "--all").Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete shc -n %s --all", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("SHC deleted", "Namespace", ns, "stdout", output) - } -} - -// SHCInNamespace returns true if SHC is present in namespace -func SHCInNamespace(ns string) bool { - output, err := exec.Command("kubectl", "get", "searchheadcluster", "-n", ns).Output() - deleted := true - if err != nil { - cmd := fmt.Sprintf("kubectl get shc -n %s", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - return deleted - } - logf.Log.Info("Output of command", "Output", string(output)) - if strings.Contains(string(output), "No resources found in default namespace") { - deleted = false - } - return deleted -} - // DeployerAppChecksum Get the checksum for each app on the deployer func DeployerAppChecksum(ctx context.Context, deployment *Deployment) map[string]string { appChecksum := make(map[string]string) @@ -143,7 +115,7 @@ func DeployerBundlePushstatus(ctx context.Context, deployment *Deployment, ns st } for appName := range appChecksum { if _, present := appBundlePush[appName]; !present { - logf.Log.Info("Deployer app not found on any members", "Appname", appName) + logf.Log.Info("Deployer app not found on any members", "appName", appName) return make(map[string]int) } } diff --git a/test/testenv/search_utils.go b/test/testenv/search_utils.go index b0e85cc66..e01f242d0 100644 --- a/test/testenv/search_utils.go +++ b/test/testenv/search_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -61,7 +61,7 @@ type SearchJobResponseResults struct { } // PerformSearchSync performs a syncronous search within splunk and returns the search results -func PerformSearchSync(ctx context.Context, podName string, search string, deployment *Deployment) (string, error) { +func PerformSearchSync(ctx context.Context, deployment *Deployment, podName string, search string) (string, error) { // Build the search curl command and send it to an instance curlCmd := "curl -ks -u" username := "admin" @@ -77,14 +77,14 @@ func PerformSearchSync(ctx context.Context, podName string, search string, deplo return "", err } - logf.Log.Info("Output of search Query", "Search", search, "Output", searchReqResp) + logf.Log.Info("Output of search Query", "search", search, "output", searchReqResp) // Since results can have multiple formats depending on the search SPL, leave this response as a string return searchReqResp, err } // PerformSearchReq makes a search request for a search to be performed. Returns a sid to be used to check for status and results -func PerformSearchReq(ctx context.Context, podName string, search string, deployment *Deployment) (string, error) { +func PerformSearchReq(ctx context.Context, deployment *Deployment, podName string, search string) (string, error) { // Build the search curl command curlCmd := "curl -ks -u" url := "https://localhost:8089/services/search/jobs" @@ -102,7 +102,7 @@ func PerformSearchReq(ctx context.Context, podName string, search string, deploy return "", err } - logf.Log.Info("Output of search Query", "Search", search, "Output", stdout) + logf.Log.Info("Output of search Query", "search", search, "output", stdout) // Get SID var searchReqResult map[string]interface{} @@ -116,7 +116,7 @@ func PerformSearchReq(ctx context.Context, podName string, search string, deploy } // GetSearchStatus checks the search status for a given -func GetSearchStatus(ctx context.Context, podName string, sid string, deployment *Deployment) (*SearchJobStatusResponse, error) { +func GetSearchStatus(ctx context.Context, deployment *Deployment, podName string, sid string) (*SearchJobStatusResponse, error) { // Build search status request curl command curlCmd := "curl -ks -u" url := "https://localhost:8089/services/search/jobs" @@ -144,7 +144,7 @@ func GetSearchStatus(ctx context.Context, podName string, sid string, deployment } // GetSearchResults retrieve the results for a given once the search status isDone == true -func GetSearchResults(ctx context.Context, podName string, sid string, deployment *Deployment) (string, error) { +func GetSearchResults(ctx context.Context, deployment *Deployment, podName string, sid string) (string, error) { // Build search results request curl command curlCmd := "curl -ks -u" url := "https://localhost:8089/services/search/jobs" diff --git a/test/testenv/secretutil.go b/test/testenv/secretutil.go index 1b9ee9ff1..c511862f0 100644 --- a/test/testenv/secretutil.go +++ b/test/testenv/secretutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,14 +46,14 @@ func GetSecretStruct(ctx context.Context, deployment *Deployment, ns string, sec secretObject := &corev1.Secret{} err := deployment.GetInstance(ctx, secretName, secretObject) if err != nil { - deployment.testenv.Log.Error(err, "Unable to get secret object", "Secret Name", secretName, "Namespace", ns) + deployment.testenv.Log.Error(err, "Unable to get secret object", "secretName", secretName, "namespace", ns) } return secretObject, err } // ModifySecretObject Modifies the secret object with given data func ModifySecretObject(ctx context.Context, deployment *Deployment, ns string, secretName string, data map[string][]byte) error { - logf.Log.Info("Modify secret object", "Secret Name", secretName, "Data", data) + logf.Log.Info("Modify secret object", "secretName", secretName, "data", data) secret := newSecretSpec(ns, secretName, data) err := deployment.UpdateCR(ctx, secret) @@ -66,7 +66,7 @@ func ModifySecretObject(ctx context.Context, deployment *Deployment, ns string, // DeleteSecretObject Deletes the entire secret object func DeleteSecretObject(ctx context.Context, deployment *Deployment, ns string, secretName string) error { - logf.Log.Info("Delete secret object", "Secret Name", secretName, "Namespace", ns) + logf.Log.Info("Delete secret object", "secretName", secretName, "namespace", ns) secret := newSecretSpec(ns, secretName, map[string][]byte{}) err := deployment.DeleteCR(ctx, secret) if err != nil { @@ -85,12 +85,12 @@ func GetMountedKey(ctx context.Context, deployment *Deployment, podName string, logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "" } - logf.Log.Info("Key found on pod", "Pod Name", podName, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Key found on pod", "podName", podName, "stdout", stdout, "stderr", stderr) return stdout } -// GetRandomeHECToken generates a random HEC token -func GetRandomeHECToken() string { +// GetRandomHECToken generates a random HEC token +func GetRandomHECToken() string { return fmt.Sprintf("%s-%s-%s-%s-%s", strings.ToUpper(RandomDNSName(8)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(12))) } @@ -99,18 +99,18 @@ func GetSecretFromServerConf(ctx context.Context, deployment *Deployment, podNam filePath := "/opt/splunk/etc/system/local/server.conf" confline, err := GetConfLineFromPod(podName, filePath, ns, configName, stanza, true) if err != nil { - logf.Log.Error(err, "Failed to get secret from pod", "Pod Name", podName, "Secret Name", configName) + logf.Log.Error(err, "Failed to get secret from pod", "podName", podName, "secretName", configName) return "", "", err } secretList := strings.Split(confline, "=") key := strings.TrimSpace(secretList[0]) - value := DecryptSplunkEncodedSecret(ctx, deployment, podName, ns, strings.TrimSpace(secretList[1])) + value := DecryptSplunkEncodedSecret(ctx, deployment, podName, strings.TrimSpace(secretList[1])) return key, value, nil } // DecryptSplunkEncodedSecret Decrypt Splunk Secret like pass4SymmKey On Given Pod -func DecryptSplunkEncodedSecret(ctx context.Context, deployment *Deployment, podName string, ns string, secretValue string) string { +func DecryptSplunkEncodedSecret(ctx context.Context, deployment *Deployment, podName string, secretValue string) string { stdin := fmt.Sprintf("/opt/splunk/bin/splunk show-decrypted --value '%s'", secretValue) command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) @@ -120,7 +120,7 @@ func DecryptSplunkEncodedSecret(ctx context.Context, deployment *Deployment, pod } logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - logf.Log.Info("Decrypted Key Value", "Decrypted Key", stdout) + logf.Log.Info("Decrypted Key Value", "decryptedKey", stdout) return strings.TrimSuffix(stdout, "\n") } @@ -157,7 +157,7 @@ func GetVersionedSecretNames(ns string, version int) []string { } } } - logf.Log.Info("Versioned Secret Objects Found in Namespace", "NameSpace", ns, "Versioned Secrets", splunkSecrets) + logf.Log.Info("Versioned Secret Objects Found in Namespace", "namespace", ns, "versionedSecrets", splunkSecrets) return splunkSecrets } @@ -194,11 +194,11 @@ func CheckSecretViaAPI(ctx context.Context, deployment *Deployment, podName stri } // GetSecretFromInputsConf gets give secret from server under given stanza -func GetSecretFromInputsConf(deployment *Deployment, podName string, ns string, configName string, stanza string) (string, string, error) { +func GetSecretFromInputsConf(podName string, ns string, configName string, stanza string) (string, string, error) { filePath := "/opt/splunk/etc/apps/splunk_httpinput/local/inputs.conf" confline, err := GetConfLineFromPod(podName, filePath, ns, configName, stanza, true) if err != nil { - logf.Log.Error(err, "Failed to get secret from pod", "Pod Name", podName, "Secret Name", configName) + logf.Log.Error(err, "Failed to get secret from pod", "podName", podName, "secretName", configName) return "", "", err } secretList := strings.Split(confline, "=") @@ -206,3 +206,15 @@ func GetSecretFromInputsConf(deployment *Deployment, podName string, ns string, value := strings.TrimSpace(secretList[1]) return key, value, nil } + +// GenerateAndApplySecretUpdate creates randomized secret data and applies it to the namespace-scoped +// secret object, returning the updated data map for subsequent verification. +func GenerateAndApplySecretUpdate(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, namespaceScopedSecretName string) (map[string][]byte, error) { + modifiedHecToken := GetRandomHECToken() + modifiedValue := RandomDNSName(10) + updatedSecretData := GetSecretDataMap(modifiedHecToken, modifiedValue, modifiedValue, modifiedValue, modifiedValue) + if err := ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData); err != nil { + return nil, fmt.Errorf("unable to update secret object: %w", err) + } + return updatedSecretData, nil +} diff --git a/test/testenv/test_setup_helpers.go b/test/testenv/test_setup_helpers.go new file mode 100644 index 000000000..6cf97dc35 --- /dev/null +++ b/test/testenv/test_setup_helpers.go @@ -0,0 +1,295 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/joho/godotenv" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" +) + +// SetupOption configures optional parameters for SetupTestCaseEnv. +type SetupOption func(*setupOptions) + +type setupOptions struct { + timeout *time.Duration +} + +// WithTimeout overrides the default test timeout for the deployment. +func WithTimeout(seconds int) SetupOption { + return func(o *setupOptions) { + d := time.Duration(seconds) * time.Second + o.timeout = &d + } +} + +// SetupTestCaseEnv creates a new test case environment and deployment for use in BeforeEach blocks. +// It also validates test prerequisites immediately to fail fast before any long operations. +func SetupTestCaseEnv(testenvInstance *TestEnv, namePrefix string, opts ...SetupOption) (*TestCaseEnv, *Deployment, error) { + var o setupOptions + for _, opt := range opts { + opt(&o) + } + + name := fmt.Sprintf("%s-%s", namePrefix+testenvInstance.GetName(), RandomDNSName(3)) + testcaseEnvInst, err := NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) + if err != nil { + return nil, nil, fmt.Errorf("unable to create testcaseenv: %w", err) + } + + deployment, err := testcaseEnvInst.NewDeployment(RandomDNSName(3), o.timeout) + if err != nil { + return nil, nil, fmt.Errorf("unable to create deployment: %w", err) + } + + if err = testcaseEnvInst.ValidateTestPrerequisites(context.TODO(), deployment); err != nil { + return nil, nil, fmt.Errorf("test prerequisites validation failed: %w", err) + } + + return testcaseEnvInst, deployment, nil +} + +// TeardownTestCaseEnv handles the common teardown logic for test case environments. +func TeardownTestCaseEnv(testcaseEnvInst *TestCaseEnv, deployment *Deployment) error { + if types.SpecState(ginkgo.CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil { + testcaseEnvInst.SkipTeardown = true + } + } + + if deployment != nil { + deployment.Teardown() + } + + if testcaseEnvInst != nil { + if err := testcaseEnvInst.Teardown(); err != nil { + return fmt.Errorf("teardown failed: %w", err) + } + } + return nil +} + +// CleanupOperatorFile deletes the test_file.img from the operator pod's app download directory +// if filePresentOnOperator is true. +func CleanupOperatorFile(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, filePresentOnOperator bool) { + if filePresentOnOperator { + opPod := testcaseEnvInst.GetOperatorPodName() + podDownloadPath := filepath.Join(AppDownloadVolume, "test_file.img") + DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) + } +} + +// TeardownAppFrameworkTestCaseEnv handles teardown for app framework tests with provider-specific +// cloud storage cleanup. cloudCleanup is called only if SkipTeardown is false. +func TeardownAppFrameworkTestCaseEnv(ctx context.Context, testcaseEnvInst *TestCaseEnv, deployment *Deployment, cloudCleanup func(), filePresentOnOperator bool) error { + if err := TeardownTestCaseEnv(testcaseEnvInst, deployment); err != nil { + return err + } + + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown && cloudCleanup != nil { + cloudCleanup() + } + + CleanupOperatorFile(ctx, deployment, testcaseEnvInst, filePresentOnOperator) + return nil +} + +// S3CloudCleanup returns a cleanup function that deletes the given files from an S3 bucket. +func S3CloudCleanup(bucket string, uploadedApps []string) func() { + return func() { + DeleteFilesOnS3(bucket, uploadedApps) + } +} + +// AzureCloudCleanup returns a cleanup function that deletes the given files from Azure Blob storage. +func AzureCloudCleanup(ctx context.Context, uploadedApps []string) func() { + return func() { + azureBlobClient := &AzureBlobClient{} + azureBlobClient.DeleteFilesOnAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, uploadedApps) + } +} + +// GCPCloudCleanup returns a cleanup function that deletes the given files from a GCP bucket. +func GCPCloudCleanup(bucket string, uploadedApps []string) func() { + return func() { + DeleteFilesOnGCP(bucket, uploadedApps) + } +} + +// LoadEnvFile traverses up the directory tree from the current working directory +// to find and load a .env file using godotenv. Returns nil if no .env file is found. +func LoadEnvFile() error { + dir, err := os.Getwd() + if err != nil { + return err + } + + for { + envFile := filepath.Join(dir, ".env") + if _, err := os.Stat(envFile); err == nil { + return godotenv.Load(envFile) + } + + parentDir := filepath.Dir(dir) + if parentDir == dir { + return nil + } + dir = parentDir + } +} + +// SetupS3AppsSuite initialises the test environment and, when running on EKS, +// downloads the V1 and V2 app sets from S3. +func SetupS3AppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string, error) { + testenvInst, err := NewDefaultTestEnv(suiteName) + if err != nil { + return nil, nil, nil, fmt.Errorf("unable to create test env: %w", err) + } + + if ClusterProvider == "eks" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + if err = DownloadFilesFromS3(testDataBucket, appDirV1, downloadDirV1, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V1 app files: %w", err) + } + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + if err = DownloadFilesFromS3(testDataBucket, appDirV2, downloadDirV2, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V2 app files: %w", err) + } + + return testenvInst, appListV1, appListV2, nil + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil, nil +} + +// CleanupLocalAppDownloads tears down the test environment and removes locally +// downloaded app directories after a suite run. +func CleanupLocalAppDownloads(testenvInst *TestEnv, dirs ...string) error { + if testenvInst != nil { + if err := testenvInst.Teardown(); err != nil { + return fmt.Errorf("teardown failed: %w", err) + } + } + for _, dir := range dirs { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("unable to delete locally downloaded app files from %s: %w", dir, err) + } + } + return nil +} + +// SetupAzureAppsSuite initialises the test environment and, when running on Azure, +// downloads the V1 and V2 app sets from Azure Blob. +func SetupAzureAppsSuite(suiteName, downloadDirV1, downloadDirV2 string) (*TestEnv, []string, []string, error) { + testenvInst, err := NewDefaultTestEnv(suiteName) + if err != nil { + return nil, nil, nil, fmt.Errorf("unable to create test env: %w", err) + } + + if ClusterProvider == "azure" { + ctx := context.TODO() + + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + containerName := "/test-data/appframework/v1apps/" + if err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV1, containerName, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V1 app files: %w", err) + } + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + containerName = "/test-data/appframework/v2apps/" + if err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV2, containerName, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V2 app files: %w", err) + } + + return testenvInst, appListV1, appListV2, nil + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil, nil +} + +// SetupGCPAppsSuite initialises the test environment and, when running on GCP, +// downloads the V1 and V2 app sets from GCS. +func SetupGCPAppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string, error) { + testenvInst, err := NewDefaultTestEnv(suiteName) + if err != nil { + return nil, nil, nil, fmt.Errorf("unable to create test env: %w", err) + } + + if ClusterProvider == "gcp" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + testenvInst.Log.Info("logging download details", "bucket", testDataBucket, "appDirV1", appDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) + if err = DownloadFilesFromGCP(testDataBucket, appDirV1, downloadDirV1, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V1 app files: %w", err) + } + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + if err = DownloadFilesFromGCP(testDataBucket, appDirV2, downloadDirV2, appFileList); err != nil { + return nil, nil, nil, fmt.Errorf("unable to download V2 app files: %w", err) + } + + return testenvInst, appListV1, appListV2, nil + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil, nil +} + +// SetupLicenseConfigMap downloads the license file from the appropriate provider +// and creates a license config map. +func SetupLicenseConfigMap(ctx context.Context, testcaseEnvInst *TestCaseEnv) error { + downloadDir := "licenseFolder" + var licenseFilePath string + var err error + + switch ClusterProvider { + case "eks": + licenseFilePath, err = DownloadLicenseFromS3Bucket() + case "azure": + licenseFilePath, err = DownloadLicenseFromAzure(ctx, downloadDir) + case "gcp": + licenseFilePath, err = DownloadLicenseFromGCPBucket() + default: + testcaseEnvInst.Log.Info("Skipping license download", "provider", ClusterProvider) + return nil + } + + if err != nil { + return fmt.Errorf("unable to download license file: %w", err) + } + + testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) + return nil +} diff --git a/test/testenv/test_workflows.go b/test/testenv/test_workflows.go new file mode 100644 index 000000000..3edc037d5 --- /dev/null +++ b/test/testenv/test_workflows.go @@ -0,0 +1,154 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package testenv + +import ( + "context" + "fmt" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// WorkflowResult contains the result of a workflow execution +type WorkflowResult struct { + Standalone *enterpriseApi.Standalone + ClusterManager *enterpriseApi.ClusterManager + IndexerCluster *enterpriseApi.IndexerCluster + SearchHeadCluster *enterpriseApi.SearchHeadCluster + MonitoringConsole *enterpriseApi.MonitoringConsole + LicenseManager *enterpriseApi.LicenseManager +} + +// RunStandaloneDeploymentWorkflow deploys a standalone instance and verifies it's ready +func (testcaseEnvInst *TestCaseEnv) RunStandaloneDeploymentWorkflow(ctx context.Context, deployment *Deployment) (*WorkflowResult, error) { + standalone, err := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, "", "") + if err != nil { + return nil, err + } + return &WorkflowResult{Standalone: standalone}, nil +} + +// RunC3DeploymentWorkflow deploys a C3 cluster (CM + IDXC + SHC) and verifies all components are ready +func (testcaseEnvInst *TestCaseEnv) RunC3DeploymentWorkflow(ctx context.Context, deployment *Deployment, indexerReplicas int, mcRef string) (*WorkflowResult, error) { + if err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, true, mcRef); err != nil { + return nil, fmt.Errorf("unable to deploy C3 cluster: %w", err) + } + + if err := testcaseEnvInst.VerifyClusterReadyAndRFSF(ctx, deployment); err != nil { + return nil, fmt.Errorf("cluster not ready: %w", err) + } + + return &WorkflowResult{}, nil +} + +// RunM4DeploymentWorkflow deploys a M4 multisite cluster and verifies all components are ready +func (testcaseEnvInst *TestCaseEnv) RunM4DeploymentWorkflow(ctx context.Context, deployment *Deployment, indexerReplicas int, siteCount int, mcRef string) (*WorkflowResult, error) { + if err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), indexerReplicas, siteCount, mcRef); err != nil { + return nil, fmt.Errorf("unable to deploy M4 cluster: %w", err) + } + + if err := testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady); err != nil { + return nil, fmt.Errorf("M4 cluster not ready: %w", err) + } + if err := testcaseEnvInst.VerifyRFSFMet(ctx, deployment); err != nil { + return nil, fmt.Errorf("RF/SF not met: %w", err) + } + + return &WorkflowResult{}, nil +} + +// RunM1DeploymentWorkflow deploys a M1 multisite indexer cluster (no SHC) and verifies components +func (testcaseEnvInst *TestCaseEnv) RunM1DeploymentWorkflow(ctx context.Context, deployment *Deployment, indexerReplicas int, siteCount int) (*WorkflowResult, error) { + if err := deployment.DeployMultisiteCluster(ctx, deployment.GetName(), indexerReplicas, siteCount, ""); err != nil { + return nil, fmt.Errorf("unable to deploy M1 cluster: %w", err) + } + + if err := testcaseEnvInst.VerifyM1ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady); err != nil { + return nil, fmt.Errorf("M1 cluster not ready: %w", err) + } + if err := testcaseEnvInst.VerifyRFSFMet(ctx, deployment); err != nil { + return nil, fmt.Errorf("RF/SF not met: %w", err) + } + + return &WorkflowResult{}, nil +} + +// RunStandaloneWithServiceAccountWorkflow deploys standalone with a service account +func (testcaseEnvInst *TestCaseEnv) RunStandaloneWithServiceAccountWorkflow(ctx context.Context, deployment *Deployment, serviceAccountName string) (*WorkflowResult, error) { + testcaseEnvInst.CreateServiceAccount(serviceAccountName) + + name := deployment.GetName() + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + ServiceAccount: serviceAccountName, + }, + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, name, spec) + if err != nil { + return nil, fmt.Errorf("unable to deploy standalone with service account: %w", err) + } + + if err = testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, name, standalone); err != nil { + return nil, fmt.Errorf("standalone not ready: %w", err) + } + + standalonePodName := fmt.Sprintf(StandalonePod, name, 0) + if err = testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(testcaseEnvInst.GetName(), standalonePodName, serviceAccountName); err != nil { + return nil, fmt.Errorf("service account not configured: %w", err) + } + + return &WorkflowResult{Standalone: standalone}, nil +} + +// RunDeleteStandaloneWorkflow deploys and deletes a standalone instance +func (testcaseEnvInst *TestCaseEnv) RunDeleteStandaloneWorkflow(ctx context.Context, deployment *Deployment) error { + result, err := testcaseEnvInst.RunStandaloneDeploymentWorkflow(ctx, deployment) + if err != nil { + return fmt.Errorf("unable to deploy standalone instance: %w", err) + } + + if err := deployment.DeleteCR(ctx, result.Standalone); err != nil { + return fmt.Errorf("unable to delete standalone instance: %w", err) + } + return nil +} + +// RunDeleteC3Workflow deploys and deletes a C3 cluster +func (testcaseEnvInst *TestCaseEnv) RunDeleteC3Workflow(ctx context.Context, deployment *Deployment, indexerReplicas int) error { + if _, err := testcaseEnvInst.RunC3DeploymentWorkflow(ctx, deployment, indexerReplicas, ""); err != nil { + return err + } + + name := deployment.GetName() + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.IndexerCluster{}, name+"-idxc"); err != nil { + return fmt.Errorf("unable to delete Indexer Cluster: %w", err) + } + + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.SearchHeadCluster{}, name+"-shc"); err != nil { + return fmt.Errorf("unable to delete Search Head Cluster: %w", err) + } + + if err := GetAndDeleteCR(ctx, deployment, &enterpriseApi.ClusterManager{}, name); err != nil { + return fmt.Errorf("unable to delete Cluster Manager: %w", err) + } + + return nil +} diff --git a/test/testenv/testcaseenv.go b/test/testenv/testcaseenv.go index 5399e56b4..89123cf0f 100644 --- a/test/testenv/testcaseenv.go +++ b/test/testenv/testcaseenv.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -116,11 +116,6 @@ func (testenv *TestCaseEnv) GetSplunkImage() string { return testenv.splunkImage } -// IsOperatorInstalledClusterWide returns if operator is installed clusterwide -func (testenv *TestCaseEnv) IsOperatorInstalledClusterWide() string { - return testenv.clusterWideOperator -} - func (testenv *TestCaseEnv) setup() error { testenv.Log.Info("testenv initializing.\n") @@ -360,7 +355,7 @@ func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { operator := &appsv1.Deployment{} err = testenv.GetKubeClient().Get(ctx, namespacedName, operator) if err != nil { - testenv.Log.Error(err, "Unable to get operator", "operator name", testenv.operatorName) + testenv.Log.Error(err, "Unable to get operator", "operatorName", testenv.operatorName) return err } @@ -385,7 +380,7 @@ func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { // update the operator deployment now err = testenv.GetKubeClient().Update(ctx, operator) if err != nil { - testenv.Log.Error(err, "Unable to update operator", "operator name", testenv.operatorName) + testenv.Log.Error(err, "Unable to update operator", "operatorName", testenv.operatorName) return err } @@ -651,12 +646,18 @@ func (testenv *TestCaseEnv) GetLMConfigMap() string { return testenv.licenseCMName } -// NewDeployment creates a new deployment -func (testenv *TestCaseEnv) NewDeployment(name string) (*Deployment, error) { +// NewDeployment creates a new deployment. If timeout is non-nil it overrides +// the default SpecifiedTestTimeout. +func (testenv *TestCaseEnv) NewDeployment(name string, timeout *time.Duration) (*Deployment, error) { + t := time.Duration(SpecifiedTestTimeout) * time.Second + if timeout != nil { + t = *timeout + } + d := Deployment{ name: testenv.GetName() + "-" + name, testenv: testenv, - testTimeout: time.Duration(SpecifiedTestTimeout) * time.Second, + testTimeout: t, } return &d, nil diff --git a/test/testenv/testenv.go b/test/testenv/testenv.go index 83576beda..e718ca9e4 100644 --- a/test/testenv/testenv.go +++ b/test/testenv/testenv.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,7 +70,7 @@ const ( // C3 deployments require bundle push across all indexers and SHC deployer which can exceed 5 minutes. AppInstallTimeout = 10 * time.Minute - // SearchHeadPod Template String for search head pod + // SearchHeadPod Template String for Search Head pod SearchHeadPod = "splunk-%s-shc-search-head-%d" // DeployerPod Template String for deployer pod @@ -79,10 +79,10 @@ const ( // StandalonePod Template String for standalone pod StandalonePod = "splunk-%s-standalone-%d" - // LicenseManagerPod Template String for standalone pod + // LicenseManagerPod Template String for License Manager pod LicenseManagerPod = "splunk-%s-license-manager-%d" - // LicenseMasterPod Template String for standalone pod + // LicenseMasterPod Template String for License Master pod LicenseMasterPod = "splunk-%s-" + splcommon.LicenseManager + "-%d" // IngestorPod Template String for ingestor pod @@ -132,11 +132,17 @@ const ( // ClusterMasterServiceName Cluster Master Service Template String ClusterMasterServiceName = "splunk-%s-cluster-master-service" - // DeployerServiceName Cluster Manager Service Template String + // DeployerServiceName Deployer Service Template String DeployerServiceName = "splunk-%s-shc-deployer-service" // CRUpdateRetryCount if CR Update fails retry these many time CRUpdateRetryCount = 10 + + // LogLineCount is the default number of log lines to ingest for test data + LogLineCount = 2000 + + // DefaultIngestIndex is the default index name used for test data ingestion + DefaultIngestIndex = "main" ) var ( diff --git a/test/testenv/util.go b/test/testenv/util.go index dbd849450..e6aa6d3c1 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,15 +23,11 @@ import ( "os" "os/exec" "path" - "reflect" - "sort" "strings" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - . "github.com/onsi/gomega" - "github.com/onsi/ginkgo/v2" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -743,7 +739,7 @@ func newStandaloneWithSpec(name, ns string, spec enterpriseApi.StandaloneSpec) * return &new } -// newMonitoringConsoleSpec returns MC Spec with given name, namespace and license manager Ref +// newMonitoringConsoleSpec returns MC Spec with given name, namespace and License Manager Ref func newMonitoringConsoleSpec(name, ns, LicenseManagerRef, splunkImage string) *enterpriseApi.MonitoringConsole { licenseMasterRef, licenseManagerRef := swapLicenseManager(name, LicenseManagerRef) @@ -871,7 +867,7 @@ func DumpGetTopPods(ns string) []string { } // GetOperatorPodName returns name of operator pod in the namespace -func GetOperatorPodName(testcaseEnvInst *TestCaseEnv) string { +func (testcaseEnvInst *TestCaseEnv) GetOperatorPodName() string { var ns string if testcaseEnvInst.clusterWideOperator != "true" { ns = testcaseEnvInst.GetName() @@ -965,7 +961,7 @@ func ExecuteCommandOnPod(ctx context.Context, deployment *Deployment, podName st logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } @@ -977,7 +973,7 @@ func ExecuteCommandOnOperatorPod(ctx context.Context, deployment *Deployment, po logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "shell", command, "command", stdin, "error", err.Error()) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } @@ -986,7 +982,7 @@ func GetConfigMap(ctx context.Context, deployment *Deployment, ns string, config configMap := &corev1.ConfigMap{} err := deployment.GetInstance(ctx, configMapName, configMap) if err != nil { - deployment.testenv.Log.Error(err, "Unable to get config map", "Config Map Name", configMap, "Namespace", ns) + deployment.testenv.Log.Error(err, "Unable to get config map", "configMapName", configMap, "namespace", ns) } return configMap, err } @@ -1057,7 +1053,7 @@ func newLicenseManagerWithGivenSpec(name, ns string, spec enterpriseApi.LicenseM return &new } -// newLicenseMasterWithGivenSpec create and initializes CR for License Manager Kind with Given Spec +// newLicenseMasterWithGivenSpec create and initializes CR for License Master Kind with Given Spec func newLicenseMasterWithGivenSpec(name, ns string, spec enterpriseApiV3.LicenseMasterSpec) *enterpriseApiV3.LicenseMaster { new := enterpriseApiV3.LicenseMaster{ TypeMeta: metav1.TypeMeta{ @@ -1115,19 +1111,9 @@ func GetDirsOrFilesInPath(ctx context.Context, deployment *Deployment, podName s return strings.Fields(stdout), err } -// CompareStringSlices checks if two string slices are matching -func CompareStringSlices(stringOne []string, stringTwo []string) bool { - if len(stringOne) != len(stringTwo) { - return false - } - sort.Strings(stringOne) - sort.Strings(stringTwo) - return reflect.DeepEqual(stringOne, stringTwo) -} - // CheckStringInSlice check if string is present in a slice func CheckStringInSlice(stringSlice []string, compString string) bool { - logf.Log.Info("Checking for string in slice", "String", compString, "String Slice", stringSlice) + logf.Log.Info("Checking for string in slice", "string", compString, "stringSlice", stringSlice) for _, item := range stringSlice { if strings.Contains(item, compString) { return true @@ -1170,18 +1156,8 @@ func GetPodUIDs(ns string) map[string]string { return splunkPodUIDs } -// DeletePod Delete pod in the namespace -func DeletePod(ns string, podName string) error { - _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() - if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) - return err - } - return nil -} - // DeleteOperatorPod Delete Operator Pod in the namespace -func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { +func (testcaseEnvInst *TestCaseEnv) DeleteOperatorPod() error { var podName string var ns string if testcaseEnvInst.clusterWideOperator != "true" { @@ -1189,11 +1165,11 @@ func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { } else { ns = "splunk-operator" } - podName = GetOperatorPodName(testcaseEnvInst) + podName = testcaseEnvInst.GetOperatorPodName() _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) + logf.Log.Error(err, "Failed to delete operator pod ", "podName", podName, "namespace", ns) return err } return nil @@ -1205,7 +1181,7 @@ func DeleteFilesOnOperatorPod(ctx context.Context, deployment *Deployment, podNa cmd := fmt.Sprintf("rm -f %s", filepath) _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to delete file on pod ", "PodName", podName, "location", filepath, "command", cmd) + logf.Log.Error(err, "Failed to delete file on pod ", "podName", podName, "location", filepath, "command", cmd) return err } } @@ -1220,9 +1196,9 @@ func DumpGetSplunkVersion(ctx context.Context, ns string, deployment *Deployment if strings.Contains(podName, filterString) { stdout, err := ExecuteCommandOnPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to get splunkd version on the pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get splunkd version on the pod", "podName", podName) } - logf.Log.Info("Splunk Version Found", "Pod Name", podName, "Version", string(stdout)) + logf.Log.Info("Splunk Version Found", "podName", podName, "version", string(stdout)) } } } @@ -1232,7 +1208,7 @@ func CreateDummyFileOnOperator(ctx context.Context, deployment *Deployment, podN cmd := fmt.Sprintf("cd %s && dd if=/dev/zero of=./%s bs=4k iflag=fullblock,count_bytes count=%s", filepath, filename, size) _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to create file on the pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to create file on the pod", "podName", podName) return err } return nil @@ -1240,10 +1216,10 @@ func CreateDummyFileOnOperator(ctx context.Context, deployment *Deployment, podN // DeleteConfigMap Delete configMap in the namespace func DeleteConfigMap(ns string, ConfigMapName string) error { - logf.Log.Info("Delete configMap", "configMap Name", ConfigMapName) + logf.Log.Info("Delete configMap", "configMapName", ConfigMapName) _, err := exec.Command("kubectl", "delete", "configmap", "-n", ns, ConfigMapName).Output() if err != nil { - logf.Log.Error(err, "Failed to delete config Map", "ConfigMap Name", ConfigMapName, "Namespace", ns) + logf.Log.Error(err, "Failed to delete config Map", "configMapName", ConfigMapName, "namespace", ns) return err } return nil @@ -1279,16 +1255,17 @@ func GetAWSEnv(podName, ns string) (string, error) { return string(output), err } -func ValidateContent(confFileContent string, listOfStringsForValidation []string, shouldContain bool) { +func ValidateContent(confFileContent string, listOfStringsForValidation []string, shouldContain bool) error { for _, str := range listOfStringsForValidation { if shouldContain { if !strings.Contains(confFileContent, str) { - Expect(confFileContent).To(ContainSubstring(str), "Failed to find string "+str+" in conf file") + return fmt.Errorf("failed to find string %q in conf file content", str) } } else { if strings.Contains(confFileContent, str) { - Expect(confFileContent).ToNot(ContainSubstring(str), "Found string "+str+" in conf file, but it should not be there") + return fmt.Errorf("found string %q in conf file, but it should not be there", str) } } } + return nil } diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index 4b3bc1331..7fc328b6e 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,12 +22,12 @@ import ( "fmt" "math/rand" "os/exec" + "reflect" "strings" "time" "sigs.k8s.io/controller-runtime/pkg/client" - gomega "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" wait "k8s.io/apimachinery/pkg/util/wait" @@ -76,164 +76,222 @@ type PodDetailsStruct struct { } `json:"status"` } +// PollConsistently verifies a condition holds for the entire duration. +// condFn should return nil if the condition holds, or an error if it fails. +func PollConsistently(duration, interval time.Duration, condFn func() error) error { + deadline := time.Now().Add(duration) + for time.Now().Before(deadline) { + if err := condFn(); err != nil { + return fmt.Errorf("consistency check failed: %w", err) + } + time.Sleep(interval) + } + return nil +} + // VerifyMonitoringConsoleReady verify Monitoring Console CR is in Ready Status and does not flip-flop -func (testenv *TestCaseEnv) VerifyMonitoringConsoleReady(ctx context.Context, deployment *Deployment, mcName string, monitoringConsole *enterpriseApi.MonitoringConsole) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifyMonitoringConsoleReady(ctx context.Context, deployment *Deployment, mcName string, monitoringConsole *enterpriseApi.MonitoringConsole) error { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, mcName, monitoringConsole) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "Phase", monitoringConsole.Status.Phase) + testenv.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "phase", monitoringConsole.Status.Phase) DumpGetPods(testenv.GetName()) - - return monitoringConsole.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return monitoringConsole.Status.Phase == enterpriseApi.PhaseReady, nil + }) + if err != nil { + return fmt.Errorf("monitoring console %s failed to reach Ready phase: %w", mcName, err) + } // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, mcName, monitoringConsole) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "monitoring-console") - return monitoringConsole.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if monitoringConsole.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("monitoring console phase flipped to %s", monitoringConsole.Status.Phase) + } + return nil + }) } // VerifyStandaloneReady verify Standalone is in ReadyStatus and does not flip-flop -func (testenv *TestCaseEnv) VerifyStandaloneReady(ctx context.Context, deployment *Deployment, deploymentName string, standalone *enterpriseApi.Standalone) { +func (testenv *TestCaseEnv) VerifyStandaloneReady(ctx context.Context, deployment *Deployment, deploymentName string, standalone *enterpriseApi.Standalone) error { // Use optimized watch to wait for Ready phase err := testenv.WatchForStandalonePhase(ctx, deployment, testenv.GetName(), standalone.Name, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "Standalone failed to reach Ready phase") + if err != nil { + return fmt.Errorf("standalone failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state err = deployment.GetInstance(ctx, standalone.Name, standalone) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("Standalone reached Ready phase", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) + if err != nil { + return fmt.Errorf("failed to get standalone instance: %w", err) + } + testenv.Log.Info("Standalone reached Ready phase", "instance", standalone.ObjectMeta.Name, "phase", standalone.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, standalone.Name, standalone) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "standalone") - return standalone.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if standalone.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("standalone phase flipped to %s", standalone.Status.Phase) + } + return nil + }) } // VerifySearchHeadClusterReady verify SHC is in READY status and does not flip-flop -func (testenv *TestCaseEnv) VerifySearchHeadClusterReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifySearchHeadClusterReady(ctx context.Context, deployment *Deployment) error { instanceName := fmt.Sprintf("%s-shc", deployment.GetName()) // Use optimized watch to wait for Ready phase (checks both Phase and DeployerPhase) err := testenv.WatchForSearchHeadClusterPhase(ctx, deployment, testenv.GetName(), instanceName, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "SearchHeadCluster failed to reach Ready phase") + if err != nil { + return fmt.Errorf("SearchHeadCluster failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state shc := &enterpriseApi.SearchHeadCluster{} err = deployment.GetInstance(ctx, instanceName, shc) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("SearchHeadCluster reached Ready phase", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase, "DeployerPhase", shc.Status.DeployerPhase) + if err != nil { + return fmt.Errorf("failed to get SearchHeadCluster instance: %w", err) + } + testenv.Log.Info("SearchHeadCluster reached Ready phase", "instance", shc.ObjectMeta.Name, "phase", shc.Status.Phase, "deployerPhase", shc.Status.DeployerPhase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, deployment.GetName(), shc) - testenv.Log.Info("Check for Consistency Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) + testenv.Log.Info("Check for Consistency Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "phase", shc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-shc-") - return shc.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if shc.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("SHC phase flipped to %s", shc.Status.Phase) + } + return nil + }) } // VerifySingleSiteIndexersReady verify single site indexers go to ready state -func (testenv *TestCaseEnv) VerifySingleSiteIndexersReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifySingleSiteIndexersReady(ctx context.Context, deployment *Deployment) error { instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) // Use optimized watch to wait for Ready phase err := testenv.WatchForIndexerClusterPhase(ctx, deployment, testenv.GetName(), instanceName, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "IndexerCluster failed to reach Ready phase") + if err != nil { + return fmt.Errorf("IndexerCluster failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state idc := &enterpriseApi.IndexerCluster{} err = deployment.GetInstance(ctx, instanceName, idc) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("IndexerCluster reached Ready phase", "instance", instanceName, "Phase", idc.Status.Phase) + if err != nil { + return fmt.Errorf("failed to get IndexerCluster instance: %w", err) + } + testenv.Log.Info("IndexerCluster reached Ready phase", "instance", instanceName, "phase", idc.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, instanceName, idc) - testenv.Log.Info("Check for Consistency indexer instance's phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Check for Consistency indexer instance's phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-idxc-indexer-") - return idc.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if idc.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("indexer phase flipped to %s", idc.Status.Phase) + } + return nil + }) } // IngestorsReady verify ingestors go to ready state -func (testenv *TestCaseEnv) VerifyIngestorReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyIngestorReady(ctx context.Context, deployment *Deployment) error { instanceName := fmt.Sprintf("%s-ingest", deployment.GetName()) // Use optimized watch to wait for Ready phase err := testenv.WatchForIngestorClusterPhase(ctx, deployment, testenv.GetName(), instanceName, enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "IngestorCluster failed to reach Ready phase") + if err != nil { + return fmt.Errorf("IngestorCluster failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state ingest := &enterpriseApi.IngestorCluster{} err = deployment.GetInstance(ctx, instanceName, ingest) - gomega.Expect(err).To(gomega.Succeed()) + if err != nil { + return fmt.Errorf("failed to get IngestorCluster instance: %w", err) + } testenv.Log.Info("IngestorCluster reached Ready phase", "instance", instanceName, "phase", ingest.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, instanceName, ingest) - testenv.Log.Info("Check for Consistency ingestor instance's phase to be ready", "instance", instanceName, "phase", ingest.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-ingest-") - - return ingest.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if ingest.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("ingestor phase flipped to %s", ingest.Status.Phase) + } + return nil + }) } // VerifyClusterManagerReady verify Cluster Manager Instance is in ready status -func (testenv *TestCaseEnv) VerifyClusterManagerReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyClusterManagerReady(ctx context.Context, deployment *Deployment) error { // Use optimized watch to wait for Ready phase err := testenv.WatchForClusterManagerPhase(ctx, deployment, testenv.GetName(), deployment.GetName(), enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "ClusterManager failed to reach Ready phase") + if err != nil { + return fmt.Errorf("ClusterManager failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state cm := &enterpriseApi.ClusterManager{} err = deployment.GetInstance(ctx, deployment.GetName(), cm) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("ClusterManager reached Ready phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + if err != nil { + return fmt.Errorf("failed to get ClusterManager instance: %w", err) + } + testenv.Log.Info("ClusterManager reached Ready phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, cluster-manager should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, deployment.GetName(), cm) - testenv.Log.Info("Check for Consistency "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("Check for Consistency "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "cluster-manager") - testenv.Log.Info("Check for Consistency cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) - return cm.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + testenv.Log.Info("Check for Consistency cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) + if cm.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("cluster manager phase flipped to %s", cm.Status.Phase) + } + return nil + }) } // VerifyClusterMasterReady verify Cluster Master Instance is in ready status -func (testenv *TestCaseEnv) VerifyClusterMasterReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyClusterMasterReady(ctx context.Context, deployment *Deployment) error { // Use optimized watch to wait for Ready phase err := testenv.WatchForClusterMasterPhase(ctx, deployment, testenv.GetName(), deployment.GetName(), enterpriseApi.PhaseReady, deployment.GetTimeout()) - gomega.Expect(err).To(gomega.Succeed(), "ClusterMaster failed to reach Ready phase") + if err != nil { + return fmt.Errorf("ClusterMaster failed to reach Ready phase: %w", err) + } // Refresh the instance to get latest state cm := &enterpriseApiV3.ClusterMaster{} err = deployment.GetInstance(ctx, deployment.GetName(), cm) - gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("ClusterMaster reached Ready phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + if err != nil { + return fmt.Errorf("failed to get ClusterMaster instance: %w", err) + } + testenv.Log.Info("ClusterMaster reached Ready phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, cluster-master should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, deployment.GetName(), cm) - testenv.Log.Info("Check for Consistency cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) - return cm.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + testenv.Log.Info("Check for Consistency cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) + if cm.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("cluster master phase flipped to %s", cm.Status.Phase) + } + return nil + }) } // VerifyIndexersReady verify indexers of all sites go to ready state -func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment *Deployment, siteCount int) { +func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment *Deployment, siteCount int) error { siteIndexerMap := map[string][]string{} for site := 1; site <= siteCount; site++ { siteName := fmt.Sprintf("site%d", site) @@ -241,50 +299,54 @@ func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment siteIndexerMap[siteName] = []string{fmt.Sprintf("splunk-%s-indexer-0", instanceName)} // Ensure indexers go to Ready phase idc := &enterpriseApi.IndexerCluster{} - gomega.Eventually(func() enterpriseApi.Phase { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, instanceName, idc) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetPods(testenv.GetName()) - - return idc.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return idc.Status.Phase == enterpriseApi.PhaseReady, nil + }) + if err != nil { + return fmt.Errorf("indexer site %s failed to reach Ready phase: %w", siteName, err) + } // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + err = PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, instanceName, idc) - testenv.Log.Info("Check for Consistency indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Check for Consistency indexer site instance phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-idxc-indexer-") - return idc.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if idc.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("indexer phase flipped to %s for site %s", idc.Status.Phase, siteName) + } + return nil + }) + if err != nil { + return err + } } + return nil } -// VerifyIndexerClusterMultisiteStatus verify indexer Cluster is configured as multisite -func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Context, deployment *Deployment, siteCount int) { +// VerifyIndexerClusterMultisiteStatus verify Indexer Cluster is configured as multisite +func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Context, deployment *Deployment, siteCount int) error { siteIndexerMap := map[string][]string{} for site := 1; site <= siteCount; site++ { siteName := fmt.Sprintf("site%d", site) instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) siteIndexerMap[siteName] = []string{fmt.Sprintf("splunk-%s-indexer-0", instanceName)} } - gomega.Eventually(func() map[string][]string { - var podName string - if strings.Contains(deployment.name, "master") { - podName = fmt.Sprintf(ClusterMasterPod, deployment.GetName()) - } else { - podName = fmt.Sprintf(ClusterManagerPod, deployment.GetName()) - } + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + podName := GetCMPodName(deployment) stdin := "curl -ks -u admin:$(cat /mnt/splunk-secrets/password) https://localhost:8089/services/cluster/manager/sites?output_mode=json" command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) if err != nil { - testenv.Log.Error(err, "Failed to execute command", "on pod", podName, "command", command) - return map[string][]string{} + testenv.Log.Error(err, "Failed to execute command", "onPod", podName, "command", command) + return false, nil } - testenv.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + testenv.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) siteIndexerResponse := ClusterManagerSitesResponse{} json.Unmarshal([]byte(stdout), &siteIndexerResponse) siteIndexerStatus := map[string][]string{} @@ -294,140 +356,152 @@ func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Cont siteIndexerStatus[site.Name] = append(siteIndexerStatus[site.Name], peer.ServerName) } } - return siteIndexerStatus - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(siteIndexerMap)) + return reflect.DeepEqual(siteIndexerStatus, siteIndexerMap), nil + }) } -// VerifyRFSFMet verify RF SF is met on cluster manager -func (testenv *TestCaseEnv) VerifyRFSFMet(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() bool { +// VerifyRFSFMet verify RF SF is met on Cluster Manager +func (testenv *TestCaseEnv) VerifyRFSFMet(ctx context.Context, deployment *Deployment) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { rfSfStatus := CheckRFSF(ctx, deployment) - testenv.Log.Info("Verifying RF SF is met", "Status", rfSfStatus) - return rfSfStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(true)) + testenv.Log.Info("Verifying RF SF is met", "status", rfSfStatus) + return rfSfStatus, nil + }) } -// VerifyNoDisconnectedSHPresentOnCM is present on cluster manager -func (testenv *TestCaseEnv) VerifyNoDisconnectedSHPresentOnCM(ctx context.Context, deployment *Deployment) { - gomega.Consistently(func() bool { +// VerifyNoDisconnectedSHPresentOnCM verifies no disconnected SH is present on Cluster Manager +func (testenv *TestCaseEnv) VerifyNoDisconnectedSHPresentOnCM(ctx context.Context, deployment *Deployment) error { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { shStatus := CheckSearchHeadRemoved(ctx, deployment) - testenv.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "Status", shStatus) - return shStatus - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) -} - -// VerifyNoSHCInNamespace verify no SHC is present in namespace -func (testenv *TestCaseEnv) VerifyNoSHCInNamespace(deployment *Deployment) { - gomega.Eventually(func() bool { - shcStatus := SHCInNamespace(testenv.GetName()) - testenv.Log.Info("Verifying no Search Head Cluster is present in namespace", "Status", shcStatus) - return shcStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(false)) + testenv.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "status", shStatus) + if !shStatus { + return fmt.Errorf("disconnected search head found on Cluster Manager") + } + return nil + }) } // VerifyLicenseManagerReady verify LM is in ready status and does not flip flop -func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deployment *Deployment) error { LicenseManager := &enterpriseApi.LicenseManager{} testenv.Log.Info("Verifying License Manager becomes READY") - gomega.Eventually(func() enterpriseApi.Phase { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, deployment.GetName(), LicenseManager) if err != nil { - return enterpriseApi.PhaseError + return false, nil } testenv.Log.Info("Waiting for License Manager instance status to be ready", - "instance", LicenseManager.ObjectMeta.Name, "Phase", LicenseManager.Status.Phase) + "instance", LicenseManager.ObjectMeta.Name, "phase", LicenseManager.Status.Phase) DumpGetPods(testenv.GetName()) - - return LicenseManager.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return LicenseManager.Status.Phase == enterpriseApi.PhaseReady, nil + }) + if err != nil { + return fmt.Errorf("license manager failed to reach Ready phase: %w", err) + } // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, deployment.GetName(), LicenseManager) - return LicenseManager.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if LicenseManager.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("license manager phase flipped to %s", LicenseManager.Status.Phase) + } + return nil + }) } // VerifyLicenseMasterReady verify LM is in ready status and does not flip flop -func (testenv *TestCaseEnv) VerifyLicenseMasterReady(ctx context.Context, deployment *Deployment) { +func (testenv *TestCaseEnv) VerifyLicenseMasterReady(ctx context.Context, deployment *Deployment) error { LicenseMaster := &enterpriseApiV3.LicenseMaster{} testenv.Log.Info("Verifying License Master becomes READY") - gomega.Eventually(func() enterpriseApi.Phase { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, deployment.GetName(), LicenseMaster) if err != nil { - return enterpriseApi.PhaseError + return false, nil } testenv.Log.Info("Waiting for License Master instance status to be ready", - "instance", LicenseMaster.ObjectMeta.Name, "Phase", LicenseMaster.Status.Phase) + "instance", LicenseMaster.ObjectMeta.Name, "phase", LicenseMaster.Status.Phase) DumpGetPods(testenv.GetName()) - - return LicenseMaster.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + return LicenseMaster.Status.Phase == enterpriseApi.PhaseReady, nil + }) + if err != nil { + return fmt.Errorf("license master failed to reach Ready phase: %w", err) + } // In a steady state, we should stay in Ready and not flip-flop around - gomega.Consistently(func() enterpriseApi.Phase { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { _ = deployment.GetInstance(ctx, deployment.GetName(), LicenseMaster) - return LicenseMaster.Status.Phase - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) + if LicenseMaster.Status.Phase != enterpriseApi.PhaseReady { + return fmt.Errorf("license master phase flipped to %s", LicenseMaster.Status.Phase) + } + return nil + }) } // VerifyLMConfiguredOnPod verify LM is configured on given POD -func VerifyLMConfiguredOnPod(ctx context.Context, deployment *Deployment, podName string) { - gomega.Consistently(func() bool { +func VerifyLMConfiguredOnPod(ctx context.Context, deployment *Deployment, podName string) error { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { lmConfigured := CheckLicenseManagerConfigured(ctx, deployment, podName) - return lmConfigured - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) + if !lmConfigured { + return fmt.Errorf("license manager not configured on pod %s", podName) + } + return nil + }) } // VerifyServiceAccountConfiguredOnPod check if given service account is configured on given pod -func (testenv *TestCaseEnv) VerifyServiceAccountConfiguredOnPod(deployment *Deployment, ns string, podName string, serviceAccount string) { - gomega.Consistently(func() bool { +func (testenv *TestCaseEnv) VerifyServiceAccountConfiguredOnPod(ns string, podName string, serviceAccount string) error { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { output, err := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() if err != nil { cmd := fmt.Sprintf("kubectl get pods -n %s %s -o json", ns, podName) testenv.Log.Error(err, "Failed to execute command", "command", cmd) - return false + return fmt.Errorf("failed to get pod %s: %w", podName, err) } restResponse := PodDetailsStruct{} err = json.Unmarshal([]byte(output), &restResponse) if err != nil { - testenv.Log.Error(err, "Failed to parse cluster Search heads") - return false + testenv.Log.Error(err, "Failed to parse cluster Search Heads") + return fmt.Errorf("failed to parse pod details: %w", err) } testenv.Log.Info("Service Account on Pod", "found", restResponse.Spec.ServiceAccount, "expected", serviceAccount) - return strings.Contains(serviceAccount, restResponse.Spec.ServiceAccount) - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) + if !strings.Contains(serviceAccount, restResponse.Spec.ServiceAccount) { + return fmt.Errorf("service account mismatch on pod %s: expected %s, found %s", podName, serviceAccount, restResponse.Spec.ServiceAccount) + } + return nil + }) } // VerifyIndexFoundOnPod verify index found on a given POD -func (testenv *TestCaseEnv) VerifyIndexFoundOnPod(ctx context.Context, deployment *Deployment, podName string, indexName string) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyIndexFoundOnPod(ctx context.Context, deployment *Deployment, podName string, indexName string) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { indexFound, _ := GetIndexOnPod(ctx, deployment, podName, indexName) testenv.Log.Info("Checking status of index on pod", "podName", podName, "indexName", indexName, "status", indexFound) - return indexFound - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return indexFound, nil + }) } // VerifyIndexConfigsMatch verify index specific config -func (testenv *TestCaseEnv) VerifyIndexConfigsMatch(ctx context.Context, deployment *Deployment, podName string, indexName string, maxGlobalDataSizeMB int, maxGlobalRawDataSizeMB int) { - gomega.Consistently(func() bool { +func (testenv *TestCaseEnv) VerifyIndexConfigsMatch(ctx context.Context, deployment *Deployment, podName string, indexName string, maxGlobalDataSizeMB int, maxGlobalRawDataSizeMB int) error { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { indexFound, data := GetIndexOnPod(ctx, deployment, podName, indexName) testenv.Log.Info("Checking status of index on pod", "podName", podName, "indexName", indexName, "status", indexFound) - if indexFound { - if data.Content.MaxGlobalDataSizeMB == maxGlobalDataSizeMB && data.Content.MaxGlobalRawDataSizeMB == maxGlobalRawDataSizeMB { - testenv.Log.Info("Checking index configs", "maxGlobalDataSizeMB", data.Content.MaxGlobalDataSizeMB, "maxGlobalRawDataSizeMB", data.Content.MaxGlobalRawDataSizeMB) - return true - } + if !indexFound { + return fmt.Errorf("index %s not found on pod %s", indexName, podName) + } + if data.Content.MaxGlobalDataSizeMB != maxGlobalDataSizeMB || data.Content.MaxGlobalRawDataSizeMB != maxGlobalRawDataSizeMB { + return fmt.Errorf("index config mismatch on pod %s: maxGlobalDataSizeMB=%d (expected %d), maxGlobalRawDataSizeMB=%d (expected %d)", + podName, data.Content.MaxGlobalDataSizeMB, maxGlobalDataSizeMB, data.Content.MaxGlobalRawDataSizeMB, maxGlobalRawDataSizeMB) } - return false - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) + testenv.Log.Info("Checking index configs", "maxGlobalDataSizeMB", data.Content.MaxGlobalDataSizeMB, "maxGlobalRawDataSizeMB", data.Content.MaxGlobalRawDataSizeMB) + return nil + }) } // VerifyIndexExistsOnS3 Verify Index Exists on S3 -func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deployment *Deployment, indexName string, podName string) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deployment *Deployment, indexName string, podName string) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { indexFound := CheckPrefixExistsOnS3(indexName) testenv.Log.Info("Checking Index on S3", "indexName", indexName, "status", indexFound) // During testing found some false failure. Rolling index buckets again to ensure data is pushed to remote storage @@ -435,95 +509,82 @@ func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deploymen testenv.Log.Info("Index NOT found. Rolling buckets again", "indexName", indexName) RollHotToWarm(ctx, deployment, podName, indexName) } - return indexFound - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) -} - -// VerifyRollingRestartFinished verify no rolling restart is active -func (testenv *TestCaseEnv) VerifyRollingRestartFinished(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() bool { - rollingRestartStatus := CheckRollingRestartStatus(ctx, deployment) - testenv.Log.Info("Rolling Restart Status", "active", rollingRestartStatus) - return rollingRestartStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(true)) + return indexFound, nil + }) } // VerifyConfOnPod Verify give conf and value on config file on pod -func (testenv *TestCaseEnv) VerifyConfOnPod(deployment *Deployment, podName string, confFilePath string, config string, value string) { - gomega.Consistently(func() bool { +func (testenv *TestCaseEnv) VerifyConfOnPod(podName string, confFilePath string, config string, value string) error { + return PollConsistently(ConsistentDuration, ConsistentPollInterval, func() error { confLine, err := GetConfLineFromPod(podName, confFilePath, testenv.GetName(), config, "", false) if err != nil { testenv.Log.Error(err, "Failed to get config on pod") - return false + return fmt.Errorf("failed to get config on pod %s: %w", podName, err) } if strings.Contains(confLine, config) && strings.Contains(confLine, value) { testenv.Log.Info("Config found", "config", config, "value", value, "confLine", confLine) - return true + return nil } testenv.Log.Info("Config NOT found") - return false - }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) + return fmt.Errorf("config %s=%s not found on pod %s", config, value, podName) + }) } // VerifySearchHeadClusterPhase verify the phase of SHC matches given phase -func (testenv *TestCaseEnv) VerifySearchHeadClusterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifySearchHeadClusterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { shc := &enterpriseApi.SearchHeadCluster{} shcName := deployment.GetName() + "-shc" err := deployment.GetInstance(ctx, shcName, shc) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "Expected", phase, "Phase", shc.Status.Phase) + testenv.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "expected", phase, "phase", shc.Status.Phase) DumpGetPods(testenv.GetName()) - - return shc.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(enterpriseApi.PhaseScalingUp)) + return shc.Status.Phase == phase, nil + }) } // VerifyIndexerClusterPhase verify the phase of idxc matches the given phase -func (testenv *TestCaseEnv) VerifyIndexerClusterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase, idxcName string) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifyIndexerClusterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase, idxcName string) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { idxc := &enterpriseApi.IndexerCluster{} err := deployment.GetInstance(ctx, idxcName, idxc) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "Expected", phase, "Phase", idxc.Status.Phase) + testenv.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "expected", phase, "phase", idxc.Status.Phase) DumpGetPods(testenv.GetName()) - - return idxc.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(phase)) + return idxc.Status.Phase == phase, nil + }) } // VerifyStandalonePhase verify the phase of Standalone CR -func (testenv *TestCaseEnv) VerifyStandalonePhase(ctx context.Context, deployment *Deployment, crName string, phase enterpriseApi.Phase) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifyStandalonePhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { standalone := &enterpriseApi.Standalone{} err := deployment.GetInstance(ctx, deployment.GetName(), standalone) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "Expected", phase, " Actual Phase", standalone.Status.Phase) + testenv.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "expected", phase, "actualPhase", standalone.Status.Phase) DumpGetPods(testenv.GetName()) - - return standalone.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(phase)) + return standalone.Status.Phase == phase, nil + }) } // VerifyMonitoringConsolePhase verify the phase of Monitoring Console CR -func (testenv *TestCaseEnv) VerifyMonitoringConsolePhase(ctx context.Context, deployment *Deployment, crName string, phase enterpriseApi.Phase) { - gomega.Eventually(func() enterpriseApi.Phase { +func (testenv *TestCaseEnv) VerifyMonitoringConsolePhase(ctx context.Context, deployment *Deployment, crName string, phase enterpriseApi.Phase) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { mc := &enterpriseApi.MonitoringConsole{} err := deployment.GetInstance(ctx, crName, mc) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "Expected", phase, " Actual Phase", mc.Status.Phase) + testenv.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "expected", phase, "actualPhase", mc.Status.Phase) DumpGetPods(testenv.GetName()) - - return mc.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(phase)) + return mc.Status.Phase == phase, nil + }) } // GetResourceVersion get resource version id @@ -560,13 +621,13 @@ func (testenv *TestCaseEnv) GetResourceVersion(ctx context.Context, deployment * } // VerifyCustomResourceVersionChanged verify the version id -func (testenv *TestCaseEnv) VerifyCustomResourceVersionChanged(ctx context.Context, deployment *Deployment, instance interface{}, resourceVersion string) { - var kind string - var newResourceVersion string - var name string - var err error +func (testenv *TestCaseEnv) VerifyCustomResourceVersionChanged(ctx context.Context, deployment *Deployment, instance interface{}, resourceVersion string) error { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + var kind string + var newResourceVersion string + var name string + var err error - gomega.Eventually(func() string { switch cr := instance.(type) { case *enterpriseApi.Standalone: err = deployment.GetInstance(ctx, cr.Name, cr) @@ -609,33 +670,32 @@ func (testenv *TestCaseEnv) VerifyCustomResourceVersionChanged(ctx context.Conte kind = cr.Kind name = cr.Name default: - return "-1" + return false, fmt.Errorf("unsupported CR type") } if err != nil { - return "-1" + return false, nil } - testenv.Log.Info("Waiting for ", kind, " CR status", "instance", name, "Not Expected", resourceVersion, " Actual Resource Version", newResourceVersion) + testenv.Log.Info("Waiting for ", kind, " CR status", "instance", name, "notExpected", resourceVersion, "actualResourceVersion", newResourceVersion) DumpGetPods(testenv.GetName()) - - return newResourceVersion - }, deployment.GetTimeout(), ShortPollInterval).ShouldNot(gomega.Equal(resourceVersion)) + return newResourceVersion != resourceVersion, nil + }) } // VerifyCPULimits verifies value of CPU limits is as expected -func (testenv *TestCaseEnv) VerifyCPULimits(deployment *Deployment, podName string, expectedCPULimits string) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyCPULimits(deployment *Deployment, podName string, expectedCPULimits string) error { + return wait.PollUntilContextTimeout(context.TODO(), PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { ns := testenv.GetName() output, err := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() if err != nil { cmd := fmt.Sprintf("kubectl get pods -n %s %s -o json", ns, podName) testenv.Log.Error(err, "Failed to execute command", "command", cmd) - return false + return false, nil } restResponse := PodDetailsStruct{} err = json.Unmarshal([]byte(output), &restResponse) if err != nil { testenv.Log.Error(err, "Failed to parse JSON") - return false + return false, nil } result := false @@ -645,25 +705,25 @@ func (testenv *TestCaseEnv) VerifyCPULimits(deployment *Deployment, podName stri testenv.Log.Info("Verifying CPU limits: ", "pod", podName, "found", restResponse.Spec.Containers[0].Resources.Limits.CPU, "expected", expectedCPULimits) } } - return result - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return result, nil + }) } // VerifyResourceConstraints verifies value of CPU limits is as expected -func (testenv *TestCaseEnv) VerifyResourceConstraints(deployment *Deployment, podName string, res corev1.ResourceRequirements) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyResourceConstraints(deployment *Deployment, podName string, res corev1.ResourceRequirements) error { + return wait.PollUntilContextTimeout(context.TODO(), PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { ns := testenv.GetName() output, err := exec.Command("kubectl", "get", "pods", "-n", ns, podName, "-o", "json").Output() if err != nil { cmd := fmt.Sprintf("kubectl get pods -n %s %s -o json", ns, podName) testenv.Log.Error(err, "Failed to execute command", "command", cmd) - return false + return false, nil } restResponse := PodDetailsStruct{} err = json.Unmarshal([]byte(output), &restResponse) if err != nil { testenv.Log.Error(err, "Failed to parse JSON") - return false + return false, nil } result := false @@ -688,45 +748,41 @@ func (testenv *TestCaseEnv) VerifyResourceConstraints(deployment *Deployment, po testenv.Log.Info("Verifying CPU limits: ", "pod", podName, "found", restResponse.Spec.Containers[i].Resources.Requests.Memory, "expected", res.Requests.Memory().String()) } } - return result - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return result, nil + }) } -// VerifyClusterManagerPhase verify phase of cluster manager -func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { +// VerifyClusterManagerPhase verify phase of Cluster Manager +func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) error { cm := &enterpriseApi.ClusterManager{} - gomega.Eventually(func() enterpriseApi.Phase { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, deployment.GetName(), cm) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) + testenv.Log.Info("Waiting for Cluster Manager Phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase, "expected", phase) DumpGetPods(testenv.GetName()) - - // Test ClusterManager Phase to see if its ready - return cm.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(phase)) + return cm.Status.Phase == phase, nil + }) } -// VerifyClusterMasterPhase verify phase of cluster manager -func (testenv *TestCaseEnv) VerifyClusterMasterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { +// VerifyClusterMasterPhase verify phase of Cluster Master +func (testenv *TestCaseEnv) VerifyClusterMasterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) error { cm := &enterpriseApiV3.ClusterMaster{} - gomega.Eventually(func() enterpriseApi.Phase { + return wait.PollUntilContextTimeout(ctx, ShortPollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { err := deployment.GetInstance(ctx, deployment.GetName(), cm) if err != nil { - return enterpriseApi.PhaseError + return false, nil } - testenv.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) + testenv.Log.Info("Waiting for Cluster Master Phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase, "expected", phase) DumpGetPods(testenv.GetName()) - - // Test ClusterManager Phase to see if its ready - return cm.Status.Phase - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(phase)) + return cm.Status.Phase == phase, nil + }) } // VerifySecretsOnPods Check whether the secret object info is mounted on given pods // Set match to true or false to indicate desired +ve or -ve match -func (testenv *TestCaseEnv) VerifySecretsOnPods(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySecretsOnPods(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) error { for _, pod := range verificationPods { for secretKey, secretValue := range data { found := false @@ -734,84 +790,102 @@ func (testenv *TestCaseEnv) VerifySecretsOnPods(ctx context.Context, deployment comparsion := bytes.Compare([]byte(currentValue), secretValue) if comparsion == 0 { found = true - testenv.Log.Info("Secret Values on POD Match", "Match Expected", match, "Pod Name ", pod, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", currentValue) + testenv.Log.Info("Secret Values on POD Match", "matchExpected", match, "podName", pod, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", currentValue) } else { - testenv.Log.Info("Secret Values on POD DONOT Match", "Match Expected", match, "Pod Name ", pod, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", currentValue) + testenv.Log.Info("Secret Values on POD DONOT Match", "matchExpected", match, "podName", pod, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", currentValue) + } + if found != match { + return fmt.Errorf("secret %s on pod %s: found=%v, expected=%v", secretKey, pod, found, match) } - gomega.Expect(found).Should(gomega.Equal(match)) } } + return nil } // VerifySecretsOnSecretObjects Compare secret value on passed in map to value present on secret object. // Set match to true or false to indicate desired +ve or -ve match -func (testenv *TestCaseEnv) VerifySecretsOnSecretObjects(ctx context.Context, deployment *Deployment, secretObjectNames []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySecretsOnSecretObjects(ctx context.Context, deployment *Deployment, secretObjectNames []string, data map[string][]byte, match bool) error { for _, secretName := range secretObjectNames { currentSecretData, err := GetSecretStruct(ctx, deployment, testenv.GetName(), secretName) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get secret struct") + if err != nil { + return fmt.Errorf("unable to get secret struct %s: %w", secretName, err) + } for secretKey, secretValue := range data { found := false secretValueOnSecretObject := currentSecretData.Data[secretKey] comparsion := bytes.Compare(secretValueOnSecretObject, secretValue) if comparsion == 0 { - testenv.Log.Info("Secret Values on Secret Object Match", "Match Expected", match, "Secret Object Name", secretName, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", string(secretValueOnSecretObject)) + testenv.Log.Info("Secret Values on Secret Object Match", "matchExpected", match, "secretObjectName", secretName, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", string(secretValueOnSecretObject)) found = true } else { - testenv.Log.Info("Secret Values on Secret Object DONOT match", "Match Expected", match, "Secret Object Name", secretName, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", string(secretValueOnSecretObject)) + testenv.Log.Info("Secret Values on Secret Object DONOT match", "matchExpected", match, "secretObjectName", secretName, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", string(secretValueOnSecretObject)) + } + if found != match { + return fmt.Errorf("secret %s on object %s: found=%v, expected=%v", secretKey, secretName, found, match) } - gomega.Expect(found).Should(gomega.Equal(match)) } } + return nil } // VerifySplunkServerConfSecrets Compare secret value on passed in map to value present on server.conf for given pods and secrets // Set match to true or false to indicate desired +ve or -ve match -func (testenv *TestCaseEnv) VerifySplunkServerConfSecrets(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySplunkServerConfSecrets(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) error { for _, podName := range verificationPods { keysToMatch := GetKeysToMatch(podName) - testenv.Log.Info("Verificaton Keys Set", "Pod Name", podName, "Keys To Compare", keysToMatch) + testenv.Log.Info("Verificaton Keys Set", "podName", podName, "keysToCompare", keysToMatch) for _, secretName := range keysToMatch { found := false stanza := SecretKeytoServerConfStanza[secretName] _, value, err := GetSecretFromServerConf(ctx, deployment, podName, testenv.GetName(), "pass4SymmKey", stanza) - gomega.Expect(err).To(gomega.Succeed(), "Secret not found in conf file", "Secret Name", secretName) + if err != nil { + return fmt.Errorf("secret %s not found in conf file on pod %s: %w", secretName, podName, err) + } comparsion := strings.Compare(value, string(data[secretName])) if comparsion == 0 { - testenv.Log.Info("Secret Values on server.conf Match", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on server.conf Match", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) found = true } else { - testenv.Log.Info("Secret Values on server.conf DONOT MATCH", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on server.conf DONOT MATCH", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) + } + if found != match { + return fmt.Errorf("secret %s on server.conf pod %s: found=%v, expected=%v", secretName, podName, found, match) } - gomega.Expect(found).Should(gomega.Equal(match)) } } + return nil } // VerifySplunkInputConfSecrets Compare secret value on passed in map to value present on input.conf for given indexer or standalone pods // Set match to true or false to indicate desired +ve or -ve match -func (testenv *TestCaseEnv) VerifySplunkInputConfSecrets(deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySplunkInputConfSecrets(verificationPods []string, data map[string][]byte, match bool) error { secretName := "hec_token" for _, podName := range verificationPods { if strings.Contains(podName, "standalone") || strings.Contains(podName, "indexer") { found := false - testenv.Log.Info("Key Verificaton", "Pod Name", podName, "Key", secretName) + testenv.Log.Info("Key Verificaton", "podName", podName, "key", secretName) stanza := SecretKeytoServerConfStanza[secretName] - _, value, err := GetSecretFromInputsConf(deployment, podName, testenv.GetName(), "token", stanza) - gomega.Expect(err).To(gomega.Succeed(), "Secret not found in conf file", "Secret Name", secretName) + _, value, err := GetSecretFromInputsConf(podName, testenv.GetName(), "token", stanza) + if err != nil { + return fmt.Errorf("secret %s not found in input.conf on pod %s: %w", secretName, podName, err) + } comparsion := strings.Compare(value, string(data[secretName])) if comparsion == 0 { - testenv.Log.Info("Secret Values on input.conf Match", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on input.conf Match", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) found = true } else { - testenv.Log.Info("Secret Values on input.conf DONOT MATCH", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on input.conf DONOT MATCH", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) + } + if found != match { + return fmt.Errorf("secret %s on input.conf pod %s: found=%v, expected=%v", secretName, podName, found, match) } - gomega.Expect(found).Should(gomega.Equal(match)) } } + return nil } // VerifySplunkSecretViaAPI check if keys can be used to access api i.e validate they are authentic -func (testenv *TestCaseEnv) VerifySplunkSecretViaAPI(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { +func (testenv *TestCaseEnv) VerifySplunkSecretViaAPI(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) error { var keysToMatch []string for _, podName := range verificationPods { if strings.Contains(podName, "standalone") || strings.Contains(podName, "indexer") { @@ -821,16 +895,19 @@ func (testenv *TestCaseEnv) VerifySplunkSecretViaAPI(ctx context.Context, deploy } for _, secretName := range keysToMatch { validKey := false - testenv.Log.Info("Key Verificaton", "Pod Name", podName, "Key", secretName) + testenv.Log.Info("Key Verificaton", "podName", podName, "key", secretName) validKey = CheckSecretViaAPI(ctx, deployment, podName, secretName, string(data[secretName])) - gomega.Expect(validKey).Should(gomega.Equal(match)) + if validKey != match { + return fmt.Errorf("secret %s via API on pod %s: valid=%v, expected=%v", secretName, podName, validKey, match) + } } } + return nil } // VerifyPVC verifies if PVC exists or not -func (testenv *TestCaseEnv) VerifyPVC(deployment *Deployment, ns string, pvcName string, expectedToExist bool, verificationTimeout time.Duration) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyPVC(pvcName string, expectedToExist bool, verificationTimeout time.Duration) error { + return wait.PollUntilContextTimeout(context.TODO(), PollInterval, verificationTimeout, true, func(ctx context.Context) (bool, error) { pvcExists := false pvcsList := DumpGetPvcs(testenv.GetName()) @@ -840,24 +917,27 @@ func (testenv *TestCaseEnv) VerifyPVC(deployment *Deployment, ns string, pvcName break } } - testenv.Log.Info("PVC Status Verified", "PVC", pvcName, "STATUS", pvcExists, "EXPECTED", expectedToExist) - return pvcExists - }, verificationTimeout, PollInterval).Should(gomega.Equal(expectedToExist)) + testenv.Log.Info("PVC Status Verified", "pvc", pvcName, "status", pvcExists, "expected", expectedToExist) + return pvcExists == expectedToExist, nil + }) } // VerifyPVCsPerDeployment verifies for a given deployment if PVCs (etc and var) exists -func (testenv *TestCaseEnv) VerifyPVCsPerDeployment(deployment *Deployment, deploymentType string, instances int, expectedtoExist bool, verificationTimeout time.Duration) { +func (testenv *TestCaseEnv) VerifyPVCsPerDeployment(deployment *Deployment, deploymentType string, instances int, expectedtoExist bool, verificationTimeout time.Duration) error { pvcKind := []string{"etc", "var"} for i := 0; i < instances; i++ { for _, pvcVolumeKind := range pvcKind { PvcName := fmt.Sprintf(PVCString, pvcVolumeKind, deployment.GetName(), deploymentType, i) - testenv.VerifyPVC(deployment, testenv.GetName(), PvcName, expectedtoExist, verificationTimeout) + if err := testenv.VerifyPVC(PvcName, expectedtoExist, verificationTimeout); err != nil { + return err + } } } + return nil } // VerifyAppInstalled verify that app of specific version is installed. Method assumes that app is installed in all CR's in namespace -func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, versionCheck bool, statusCheck string, checkupdated bool, clusterWideInstall bool) { +func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, versionCheck bool, statusCheck string, checkupdated bool, clusterWideInstall bool) error { // Fail-fast test: check first pod and first app before checking all pods if len(pods) > 0 && len(apps) > 0 { testenv.Log.Info("Running fail-fast test on first pod before checking all pods", "pod", pods[0], "app", apps[0]) @@ -866,7 +946,7 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * status, versionInstalled, err := GetPodAppStatus(ctx, deployment, firstPod, ns, firstApp, clusterWideInstall) if err != nil { - gomega.Expect(err).To(gomega.Succeed(), fmt.Sprintf("Test failed - app %s not accessible on pod %s. This indicates a fundamental issue. Skipping remaining checks.", firstApp, firstPod)) + return fmt.Errorf("test failed - app %s not accessible on pod %s: %w", firstApp, firstPod, err) } testenv.Log.Info("Test passed - app is accessible", "pod", firstPod, "app", firstApp, "status", status, "version", versionInstalled) testenv.Log.Info("Proceeding with full verification of all pods and apps") @@ -876,7 +956,9 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * for _, appName := range apps { status, versionInstalled, err := GetPodAppStatus(ctx, deployment, podName, ns, appName, clusterWideInstall) testenv.Log.Info("App details", "app", appName, "status", status, "version", versionInstalled, "error", err) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get app status on pod ") + if err != nil { + return fmt.Errorf("unable to get app status on pod %s: %w", podName, err) + } comparison := strings.EqualFold(status, statusCheck) //Check the app is installed on specific pods and un-installed on others for cluster-wide install var check bool @@ -884,7 +966,9 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * if strings.Contains(podName, "-indexer-") || strings.Contains(podName, "-search-head-") { check = true testenv.Log.Info("App Install Check", "pod", podName, "app", appName, "expected", check, "found", comparison, "scope:cluster", clusterWideInstall) - gomega.Expect(comparison).Should(gomega.Equal(check)) + if comparison != check { + return fmt.Errorf("app %s install check failed on pod %s: expected=%v, found=%v", appName, podName, check, comparison) + } } } else { // For local install check pods individually @@ -894,7 +978,9 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * check = true } testenv.Log.Info("App Install Check", "pod", podName, "app", appName, "expected", check, "found", comparison, "scope:cluster", clusterWideInstall) - gomega.Expect(comparison).Should(gomega.Equal(check)) + if comparison != check { + return fmt.Errorf("app %s install check failed on pod %s: expected=%v, found=%v", appName, podName, check, comparison) + } } if versionCheck { @@ -907,15 +993,18 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * expectedVersion = AppInfo[appName]["V1"] } testenv.Log.Info("Verify app", "pod", podName, "app", appName, "expectedVersion", expectedVersion, "versionInstalled", versionInstalled, "updated", checkupdated) - gomega.Expect(versionInstalled).Should(gomega.Equal(expectedVersion)) + if versionInstalled != expectedVersion { + return fmt.Errorf("app %s version mismatch on pod %s: expected=%s, found=%s", appName, podName, expectedVersion, versionInstalled) + } } } } } + return nil } // VerifyAppsCopied verify that apps are copied to correct location based on POD. Set checkAppDirectory false to verify app is not copied. -func (testenv *TestCaseEnv) VerifyAppsCopied(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, checkAppDirectory bool, scope string) { +func (testenv *TestCaseEnv) VerifyAppsCopied(ctx context.Context, deployment *Deployment, pods []string, apps []string, checkAppDirectory bool, scope string) error { for _, podName := range pods { path := "etc/apps" @@ -929,157 +1018,201 @@ func (testenv *TestCaseEnv) VerifyAppsCopied(ctx context.Context, deployment *De path = splcommon.PeerAppsLoc } } - testenv.VerifyAppsInFolder(ctx, deployment, ns, podName, apps, path, checkAppDirectory) + if err := testenv.VerifyAppsInFolder(ctx, deployment, podName, apps, path, checkAppDirectory); err != nil { + return err + } } + return nil } // VerifyAppsInFolder verify that apps are present in folder. Set checkAppDirectory false to verify app is not copied. -func (testenv *TestCaseEnv) VerifyAppsInFolder(ctx context.Context, deployment *Deployment, ns string, podName string, apps []string, path string, checkAppDirectory bool) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyAppsInFolder(ctx context.Context, deployment *Deployment, podName string, apps []string, path string, checkAppDirectory bool) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { // Using checkAppDirectory here to get all files in case of negative check. GetDirsOrFilesInPath will return files/directory when checkAppDirecotry is FALSE appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, checkAppDirectory) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "Pod", podName) + if err != nil { + return false, fmt.Errorf("unable to get apps on pod %s: %w", podName, err) + } for _, app := range apps { folderName := app + "/" found := CheckStringInSlice(appList, folderName) testenv.Log.Info("App check", "pod", podName, "folderName", folderName, "path", path, "status", found) if found != checkAppDirectory { - return false + return false, nil } } - return true - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return true, nil + }) } // VerifyAppsDownloadedOnContainer verify that apps are downloaded by init container -func (testenv *TestCaseEnv) VerifyAppsDownloadedOnContainer(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, path string) { +func (testenv *TestCaseEnv) VerifyAppsDownloadedOnContainer(ctx context.Context, deployment *Deployment, pods []string, apps []string, path string) error { for _, podName := range pods { appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, false) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "Pod", podName) + if err != nil { + return fmt.Errorf("unable to get apps on pod %s: %w", podName, err) + } for _, app := range apps { found := CheckStringInSlice(appList, app) - testenv.Log.Info("Check App files present on the pod", "Pod Name", podName, "App Name", app, "directory", path, "Status", found) - gomega.Expect(found).Should(gomega.Equal(true)) + testenv.Log.Info("Check App files present on the pod", "podName", podName, "appName", app, "directory", path, "status", found) + if !found { + return fmt.Errorf("app %s not found on pod %s in path %s", app, podName, path) + } } } + return nil } // VerifyAppsPackageDeletedOnOperatorContainer verify that apps are deleted by container -func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnOperatorContainer(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, path string) { +func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnOperatorContainer(ctx context.Context, deployment *Deployment, pods []string, apps []string, path string) error { for _, podName := range pods { for _, app := range apps { - gomega.Eventually(func() bool { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { appList, err := GetOperatorDirsOrFilesInPath(ctx, deployment, podName, path, false) if err != nil { - testenv.Log.Error(err, "Unable to get apps on operator pod", "Pod", podName) - return true + testenv.Log.Error(err, "Unable to get apps on operator pod", "pod", podName) + return false, nil } found := CheckStringInSlice(appList, app+"_") testenv.Log.Info(fmt.Sprintf("Check App package deleted on the pod %s. App Name %s. Directory %s, Status %t", podName, app, path, found)) - return found - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(false)) + return !found, nil + }) + if err != nil { + return fmt.Errorf("app package %s not deleted on operator pod %s: %w", app, podName, err) + } } } + return nil } // VerifyAppsPackageDeletedOnContainer verify that apps are deleted by container -func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnContainer(ctx context.Context, deployment *Deployment, ns string, pods []string, apps []string, path string) { +func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnContainer(ctx context.Context, deployment *Deployment, pods []string, apps []string, path string) error { for _, podName := range pods { for _, app := range apps { - gomega.Eventually(func() bool { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, false) if err != nil { - testenv.Log.Error(err, "Unable to get apps on pod", "Pod", podName) - return true + testenv.Log.Error(err, "Unable to get apps on pod", "pod", podName) + return false, nil } found := CheckStringInSlice(appList, app+"_") testenv.Log.Info(fmt.Sprintf("Check App package deleted on the pod %s. App Name %s. Directory %s, Status %t", podName, app, path, found)) - return found - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(false)) + return !found, nil + }) + if err != nil { + return fmt.Errorf("app package %s not deleted on pod %s: %w", app, podName, err) + } } } + return nil } // VerifyAppListPhase verify given app Phase has completed for the given list of apps for given CR Kind -func (testenv *TestCaseEnv) VerifyAppListPhase(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, phase enterpriseApi.AppPhaseType, appList []string) { +func (testenv *TestCaseEnv) VerifyAppListPhase(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, phase enterpriseApi.AppPhaseType, appList []string) error { if phase == enterpriseApi.PhaseDownload || phase == enterpriseApi.PhasePodCopy { for _, appName := range appList { testenv.Log.Info(fmt.Sprintf("Check App Status for CR %s NAME %s APP NAME %s Expected Phase not to be %s", crKind, name, appName, phase)) - gomega.Eventually(func() enterpriseApi.AppPhaseType { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) if err != nil { testenv.Log.Error(err, "Failed to get app deployment info") - return phase // Continue polling + return false, nil // Continue polling } if appDeploymentInfo.AppName == "" { testenv.Log.Info(fmt.Sprintf("App deployment info not found yet for app %s (CR %s/%s, AppSource %s), continuing to poll", appName, crKind, name, appSourceName)) - return phase // Continue polling + return false, nil // Continue polling } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase should not be %s", crKind, name, appName, phase), "Actual Phase", appDeploymentInfo.PhaseInfo.Phase, "App State", appDeploymentInfo) - return appDeploymentInfo.PhaseInfo.Phase - }, deployment.GetTimeout(), PollInterval).ShouldNot(gomega.Equal(phase)) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase should not be %s", crKind, name, appName, phase), "actualPhase", appDeploymentInfo.PhaseInfo.Phase, "appState", appDeploymentInfo) + return appDeploymentInfo.PhaseInfo.Phase != phase, nil + }) + if err != nil { + return fmt.Errorf("app %s on CR %s/%s did not move past phase %s: %w", appName, crKind, name, phase, err) + } } } else { for _, appName := range appList { testenv.Log.Info(fmt.Sprintf("Check App Status for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase)) - gomega.Eventually(func() enterpriseApi.AppPhaseType { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) if err != nil { testenv.Log.Error(err, "Failed to get app deployment info") - return enterpriseApi.PhaseDownload // Continue polling + return false, nil // Continue polling } if appDeploymentInfo.AppName == "" { testenv.Log.Info(fmt.Sprintf("App deployment info not found yet for app %s (CR %s/%s, AppSource %s), continuing to poll", appName, crKind, name, appSourceName)) - return enterpriseApi.PhaseDownload // Continue polling + return false, nil // Continue polling } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase), "Actual Phase", appDeploymentInfo.PhaseInfo.Phase, "App Phase Status", appDeploymentInfo.PhaseInfo.Status, "App State", appDeploymentInfo) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase), "actualPhase", appDeploymentInfo.PhaseInfo.Phase, "appPhaseStatus", appDeploymentInfo.PhaseInfo.Status, "appState", appDeploymentInfo) if appDeploymentInfo.PhaseInfo.Status != enterpriseApi.AppPkgInstallComplete { - testenv.Log.Info("Phase Install Not Complete.", "Phase Found", appDeploymentInfo.PhaseInfo.Phase, "Phase Status Found", appDeploymentInfo.PhaseInfo.Status) - return enterpriseApi.PhaseDownload + testenv.Log.Info("Phase Install Not Complete.", "phaseFound", appDeploymentInfo.PhaseInfo.Phase, "phaseStatusFound", appDeploymentInfo.PhaseInfo.Status) + return false, nil } - return appDeploymentInfo.PhaseInfo.Phase - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) + return appDeploymentInfo.PhaseInfo.Phase == phase, nil + }) + if err != nil { + return fmt.Errorf("app %s on CR %s/%s did not reach phase %s: %w", appName, crKind, name, phase, err) + } } } + return nil } // VerifyAppState verify given app state is in between states passed as parameters, i.e when Status is between 101 and 303 we would pass enterpriseApi.AppPkgInstallComplete and enterpriseApi.AppPkgPodCopyComplete -func (testenv *TestCaseEnv) VerifyAppState(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string, appStateFinal enterpriseApi.AppPhaseStatusType, appStateInitial enterpriseApi.AppPhaseStatusType) { +func (testenv *TestCaseEnv) VerifyAppState(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string, appStateFinal enterpriseApi.AppPhaseStatusType, appStateInitial enterpriseApi.AppPhaseStatusType) error { for _, appName := range appList { - gomega.Eventually(func() enterpriseApi.AppPhaseStatusType { - appDeploymentInfo, _ := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) - return appDeploymentInfo.PhaseInfo.Status - }, deployment.GetTimeout(), PollInterval).Should(gomega.BeNumerically("~", appStateFinal, appStateInitial)) //Check status value is between appStateInitial and appStateFinal + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, _ := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) + status := appDeploymentInfo.PhaseInfo.Status + // Check status value is approximately between appStateInitial and appStateFinal + diff := status - appStateFinal + if diff < 0 { + diff = -diff + } + return diff <= appStateInitial, nil + }) + if err != nil { + return fmt.Errorf("app %s state not in expected range: %w", appName, err) + } } + return nil } // WaitForAppInstall waits until an app is correctly installed (having status equal to 303) -func (testenv *TestCaseEnv) WaitForAppInstall(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string) { +func (testenv *TestCaseEnv) WaitForAppInstall(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, appList []string) error { for _, appName := range appList { - gomega.Eventually(func() enterpriseApi.AppPhaseStatusType { - appDeploymentInfo, _ := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) - return appDeploymentInfo.PhaseInfo.Status - }, deployment.GetTimeout(), PollInterval).Should(gomega.BeEquivalentTo(enterpriseApi.AppPkgInstallComplete)) + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, _ := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) + return appDeploymentInfo.PhaseInfo.Status == enterpriseApi.AppPkgInstallComplete, nil + }) + if err != nil { + return fmt.Errorf("app %s did not reach install complete status: %w", appName, err) + } } - + return nil } // VerifyPodsInMCConfigMap checks if given pod names are present in given KEY of given MC's Config Map -func (testenv *TestCaseEnv) VerifyPodsInMCConfigMap(ctx context.Context, deployment *Deployment, pods []string, key string, mcName string, expected bool) { +func (testenv *TestCaseEnv) VerifyPodsInMCConfigMap(ctx context.Context, deployment *Deployment, pods []string, key string, mcName string, expected bool) error { // Get contents of MC config map mcConfigMap, err := GetMCConfigMap(ctx, deployment, testenv.GetName(), mcName) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get MC config map") + if err != nil { + return fmt.Errorf("unable to get MC config map: %w", err) + } for _, podName := range pods { - testenv.Log.Info("Checking for POD on MC Config Map", "POD Name", podName, "DATA", mcConfigMap.Data) - gomega.Expect(expected).To(gomega.Equal(CheckPodNameInString(podName, mcConfigMap.Data[key])), "Verify Pod in MC Config Map. Pod Name %s.", podName) + testenv.Log.Info("Checking for POD on MC Config Map", "podName", podName, "data", mcConfigMap.Data) + found := CheckPodNameInString(podName, mcConfigMap.Data[key]) + if found != expected { + return fmt.Errorf("verify pod in MC Config Map failed: pod %s, found=%v, expected=%v", podName, found, expected) + } } + return nil } // VerifyPodsInMCConfigString checks if given pod names are present in given KEY of given MC's Config Map -func (testenv *TestCaseEnv) VerifyPodsInMCConfigString(ctx context.Context, deployment *Deployment, pods []string, mcName string, expected bool, checkPodIP bool) { +func (testenv *TestCaseEnv) VerifyPodsInMCConfigString(ctx context.Context, pods []string, mcName string, expected bool, checkPodIP bool) error { for _, podName := range pods { - testenv.Log.Info("Checking pod configured in MC POD Peers String", "Pod Name", podName) + testenv.Log.Info("Checking pod configured in MC POD Peers String", "podName", podName) var found bool if checkPodIP { podIP := GetPodIP(testenv.GetName(), podName) @@ -1087,21 +1220,25 @@ func (testenv *TestCaseEnv) VerifyPodsInMCConfigString(ctx context.Context, depl } else { found = CheckPodNameOnMC(testenv.GetName(), mcName, podName) } - gomega.Expect(expected).To(gomega.Equal(found), "Verify Pod in MC Config String. Pod Name %s.", podName) + if found != expected { + return fmt.Errorf("verify pod in MC Config String failed: pod %s, found=%v, expected=%v", podName, found, expected) + } } + return nil } // VerifyClusterManagerBundlePush verify that bundle push was pushed on all indexers -func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, deployment *Deployment, ns string, replicas int, previousBundleHash string) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, deployment *Deployment, replicas int, previousBundleHash string) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { // Get Bundle status and check that each pod has successfully deployed the latest bundle - clusterManagerBundleStatus := CMBundlePushstatus(ctx, deployment, previousBundleHash, "cmanager") + cmEndpoint := "cmanager" if strings.Contains(deployment.GetName(), "master") { - clusterManagerBundleStatus = CMBundlePushstatus(ctx, deployment, previousBundleHash, "cmaster") + cmEndpoint = "cmaster" } + clusterManagerBundleStatus := CMBundlePushstatus(ctx, deployment, previousBundleHash, cmEndpoint) if len(clusterManagerBundleStatus) < replicas { - testenv.Log.Info("Bundle push on Pod not complete on all pods", "Pod with bundle push", clusterManagerBundleStatus) - return false + testenv.Log.Info("Bundle push on Pod not complete on all pods", "podWithBundlePush", clusterManagerBundleStatus) + return false, nil } clusterPodNames := DumpGetPods(testenv.GetName()) @@ -1109,43 +1246,43 @@ func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, if strings.Contains(podName, "-indexer-") { if _, present := clusterManagerBundleStatus[podName]; present { if clusterManagerBundleStatus[podName] != "Up" { - testenv.Log.Info("Bundle push on Pod not complete", "Pod Name", podName, "Status", clusterManagerBundleStatus[podName]) - return false + testenv.Log.Info("Bundle push on Pod not complete", "podName", podName, "status", clusterManagerBundleStatus[podName]) + return false, nil } } else { - testenv.Log.Info("Bundle push not found on pod", "Podname", podName) - return false + testenv.Log.Info("Bundle push not found on pod", "podName", podName) + return false, nil } } } - return true - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return true, nil + }) } // VerifyDeployerBundlePush verify that bundle push was pushed on all search heads -func (testenv *TestCaseEnv) VerifyDeployerBundlePush(ctx context.Context, deployment *Deployment, ns string, replicas int) { - gomega.Eventually(func() bool { +func (testenv *TestCaseEnv) VerifyDeployerBundlePush(ctx context.Context, deployment *Deployment, ns string, replicas int) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { deployerAppPushStatus := DeployerBundlePushstatus(ctx, deployment, ns) if len(deployerAppPushStatus) == 0 { testenv.Log.Info("Bundle push not complete on all pods") DumpGetPods(testenv.GetName()) - return false + return false, nil } for appName, val := range deployerAppPushStatus { if val < replicas { - testenv.Log.Info("Bundle push not complete on all pods for", "App Name", appName, "Replicas with bundle push", val, "Expected replicas", replicas) + testenv.Log.Info("Bundle push not complete on all pods for", "appName", appName, "replicasWithBundlePush", val, "expectedReplicas", replicas) DumpGetPods(testenv.GetName()) - return false + return false, nil } } - return true - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return true, nil + }) } // VerifyNoPodResetByUID verify that no pod reset during App install by comparing pod UIDs -func (testenv *TestCaseEnv) VerifyNoPodResetByUID(ctx context.Context, deployment *Deployment, podUIDMap map[string]string, podToSkip []string) { +func (testenv *TestCaseEnv) VerifyNoPodResetByUID(ctx context.Context, podUIDMap map[string]string, podToSkip []string) error { if podUIDMap == nil { testenv.Log.Info("podUIDMap is empty. Skipping validation") } else { @@ -1157,78 +1294,91 @@ func (testenv *TestCaseEnv) VerifyNoPodResetByUID(ctx context.Context, deploymen testenv.Log.Info("Checking Pod reset for Pod Name", "podName", podName, "currentUID", currentUID) if previousUID, ok := podUIDMap[podName]; ok { if !CheckStringInSlice(podToSkip, podName) { - gomega.Expect(currentUID).To(gomega.Equal(previousUID), "Pod reset was detected. Pod Name %s. Current Pod UID %s. Previous Pod UID %s", podName, currentUID, previousUID) + if currentUID != previousUID { + return fmt.Errorf("pod reset was detected. Pod Name %s. Current Pod UID %s. Previous Pod UID %s", podName, currentUID, previousUID) + } } } } } + return nil } -// WaitForSplunkPodCleanup Wait for cleanup to happend -func (testenv *TestCaseEnv) WaitForSplunkPodCleanup(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() int { +// WaitForSplunkPodCleanup Wait for cleanup to happen +func (testenv *TestCaseEnv) WaitForSplunkPodCleanup(ctx context.Context, deployment *Deployment) error { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { testenv.Log.Info("Waiting for Splunk Pods to be deleted before running test") - return len(DumpGetPods(testenv.GetName())) - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(0)) + return len(DumpGetPods(testenv.GetName())) == 0, nil + }) } // WaitforAppInstallState Wait for App to reach state specified in conf file -func (testenv *TestCaseEnv) WaitforAppInstallState(ctx context.Context, deployment *Deployment, podNames []string, ns string, appName string, newState string, clusterWideInstall bool) { +func (testenv *TestCaseEnv) WaitforAppInstallState(ctx context.Context, deployment *Deployment, podNames []string, ns string, appName string, newState string, clusterWideInstall bool) error { testenv.Log.Info("Retrieve App state on pod") for _, podName := range podNames { - gomega.Eventually(func() string { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { status, _, err := GetPodAppStatus(ctx, deployment, podName, ns, appName, clusterWideInstall) testenv.Log.Info("App details", "app", appName, "status", status, "error", err, "podName", podName) - return status - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(strings.ToUpper(newState))) + return status == strings.ToUpper(newState), nil + }) + if err != nil { + return fmt.Errorf("app %s did not reach state %s on pod %s: %w", appName, newState, podName, err) + } } + return nil } // VerifyAppRepoState verify given app repo state is equal to given value for app for given CR Kind -func (testenv *TestCaseEnv) VerifyAppRepoState(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, repoValue int, appName string) { +func (testenv *TestCaseEnv) VerifyAppRepoState(ctx context.Context, deployment *Deployment, name string, crKind string, appSourceName string, repoValue int, appName string) error { testenv.Log.Info("Check for app repo state in CR") - gomega.Eventually(func() int { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, name, crKind, appSourceName, appName) + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, name, crKind, appSourceName, appName) if err != nil { testenv.Log.Error(err, "Failed to get app deployment info") - return 0 + return false, nil } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected repo value %d", crKind, name, appName, repoValue), "Actual Value", appDeploymentInfo.RepoState, "App State", appDeploymentInfo) - return int(appDeploymentInfo.RepoState) - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(repoValue)) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected repo value %d", crKind, name, appName, repoValue), "actualValue", appDeploymentInfo.RepoState, "appState", appDeploymentInfo) + return int(appDeploymentInfo.RepoState) == repoValue, nil + }) } // VerifyIsDeploymentInProgressFlagIsSet verify IsDeploymentInProgress flag is set to true -func (testenv *TestCaseEnv) VerifyIsDeploymentInProgressFlagIsSet(ctx context.Context, deployment *Deployment, name string, crKind string) { - testenv.Log.Info("Check IsDeploymentInProgress Flag is set", "CR NAME", name, "CR Kind", crKind) - gomega.Eventually(func() bool { - isDeploymentInProgress, err := GetIsDeploymentInProgressFlag(ctx, deployment, testenv, name, crKind) +func (testenv *TestCaseEnv) VerifyIsDeploymentInProgressFlagIsSet(ctx context.Context, deployment *Deployment, name string, crKind string) error { + testenv.Log.Info("Check IsDeploymentInProgress Flag is set", "crName", name, "crKind", crKind) + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { + isDeploymentInProgress, err := testenv.GetIsDeploymentInProgressFlag(ctx, deployment, name, crKind) if err != nil { testenv.Log.Error(err, "Failed to get isDeploymentInProgress Flag") - return false + return false, nil } - testenv.Log.Info("IsDeploymentInProgress Flag status found", "CR NAME", name, "CR Kind", crKind, "IsDeploymentInProgress", isDeploymentInProgress) - return isDeploymentInProgress - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + testenv.Log.Info("IsDeploymentInProgress Flag status found", "crName", name, "crKind", crKind, "isDeploymentInProgress", isDeploymentInProgress) + return isDeploymentInProgress, nil + }) } // VerifyFilesInDirectoryOnPod verify that files are present in folder. -func (testenv *TestCaseEnv) VerifyFilesInDirectoryOnPod(ctx context.Context, deployment *Deployment, podNames []string, files []string, path string, checkDirectory bool, checkPresent bool) { +func (testenv *TestCaseEnv) VerifyFilesInDirectoryOnPod(ctx context.Context, deployment *Deployment, podNames []string, files []string, path string, checkDirectory bool, checkPresent bool) error { for _, podName := range podNames { - gomega.Eventually(func() bool { + err := wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { // Using checkDirectory here to get all files in case of negative check. GetDirsOrFilesInPath will return files/directory when checkDirecotry is FALSE filelist, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, checkDirectory) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get files on pod", "Pod", podName) + if err != nil { + return false, fmt.Errorf("unable to get files on pod %s: %w", podName, err) + } for _, file := range files { found := CheckStringInSlice(filelist, file) testenv.Log.Info("File check", "pod", podName, "filename", file, "path", path, "status", found) if found != checkPresent { - return false + return false, nil } } - return true - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return true, nil + }) + if err != nil { + return err + } } + return nil } func (testenv *TestCaseEnv) GetTelemetryLastSubmissionTime(ctx context.Context, deployment *Deployment) string { @@ -1263,16 +1413,16 @@ func (testenv *TestCaseEnv) GetTelemetryLastSubmissionTime(ctx context.Context, } // VerifyTelemetry checks that the telemetry ConfigMap has a non-empty lastTransmission field in its status key. -func (testenv *TestCaseEnv) VerifyTelemetry(ctx context.Context, deployment *Deployment, prevVal string) { +func (testenv *TestCaseEnv) VerifyTelemetry(ctx context.Context, deployment *Deployment, prevVal string) error { testenv.Log.Info("VerifyTelemetry: start") - gomega.Eventually(func() bool { + return wait.PollUntilContextTimeout(ctx, PollInterval, deployment.GetTimeout(), true, func(ctx context.Context) (bool, error) { currentVal := testenv.GetTelemetryLastSubmissionTime(ctx, deployment) if currentVal != "" && currentVal != prevVal { testenv.Log.Info("VerifyTelemetry: success", "previous", prevVal, "current", currentVal) - return true + return true, nil } - return false - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) + return false, nil + }) } // TriggerTelemetrySubmission updates or adds the 'test_submission' key in the telemetry ConfigMap with a JSON value containing a random number. @@ -1370,7 +1520,7 @@ func (testenv *TestCaseEnv) WaitForPodsInMCConfigMap(ctx context.Context, deploy } // WaitForPodsInMCConfigString waits for pods to appear in MC config string -func (testenv *TestCaseEnv) WaitForPodsInMCConfigString(ctx context.Context, deployment *Deployment, pods []string, mcName string, expected bool, checkPodIP bool, timeout time.Duration) error { +func (testenv *TestCaseEnv) WaitForPodsInMCConfigString(ctx context.Context, pods []string, mcName string, expected bool, checkPodIP bool, timeout time.Duration) error { return wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { for _, podName := range pods { var found bool @@ -1421,7 +1571,7 @@ func (testenv *TestCaseEnv) WaitForIndexerClusterPhase(ctx context.Context, depl // WaitForSearchResultsNonEmpty waits for search results to return a non-empty "result" field func WaitForSearchResultsNonEmpty(ctx context.Context, deployment *Deployment, podName string, searchString string, timeout time.Duration) error { return wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { - searchResultsResp, err := PerformSearchSync(ctx, podName, searchString, deployment) + searchResultsResp, err := PerformSearchSync(ctx, deployment, podName, searchString) if err != nil { return false, nil } @@ -1538,7 +1688,7 @@ func (testenv *TestCaseEnv) WaitForAppRepoStateChange(ctx context.Context, deplo } } - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, crName, crKind, appSourceName, lookupAppName) + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, crName, crKind, appSourceName, lookupAppName) if err != nil { testenv.Log.Info("Failed to get app deployment info while waiting for repo state change", "app", appName, "error", err) return false, nil @@ -1564,3 +1714,374 @@ func (testenv *TestCaseEnv) WaitForAppRepoStateChange(ctx context.Context, deplo return false, nil }) } + +// VerifyC3ClusterPVCs verifies that PVCs for SHC, deployer, indexers, and cluster manager exist or are deleted. +func VerifyC3ClusterPVCs(testcaseEnvInst *TestCaseEnv, deployment *Deployment, clusterManagerType string, exists bool, timeout time.Duration) error { + if err := testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, exists, timeout); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, exists, timeout); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, exists, timeout); err != nil { + return err + } + return testcaseEnvInst.VerifyPVCsPerDeployment(deployment, clusterManagerType, 1, exists, timeout) +} + +// VerifyM4ClusterAndRFSF verifies cluster manager and multisite cluster are ready and RF/SF is met. +func VerifyM4ClusterAndRFSF(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig, siteCount int) error { + if err := config.ClusterManagerReady(ctx, deployment, testcaseEnvInst); err != nil { + return err + } + if err := testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount); err != nil { + return err + } + if err := testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount); err != nil { + return err + } + if err := testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment); err != nil { + return err + } + return testcaseEnvInst.VerifyRFSFMet(ctx, deployment) +} + +// VerifyLMAppsOnPod verifies that apps are copied and installed on the License Manager pod. +// The updated flag controls whether apps are expected to be updated versions. +func VerifyLMAppsOnPod(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, testenvInstance *TestEnv, podName []string, appList []string, updated bool) error { + if err := testcaseEnvInst.VerifyAppsCopied(ctx, deployment, podName, appList, true, enterpriseApi.ScopeLocal); err != nil { + return err + } + return testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appList, updated, "enabled", updated, false) +} + +// VerifyLMConfiguredOnCluster verifies that the License Manager is configured on +// the given indexer pods, on search head pods, and on the Monitoring Console. +func VerifyLMConfiguredOnCluster(ctx context.Context, deployment *Deployment, indexerPods []string) error { + shPods := GeneratePodNameSlice(SearchHeadPod, deployment.GetName(), 3, false, 0) + if err := VerifyLMConfiguredOnPods(ctx, deployment, append(indexerPods, shPods...)); err != nil { + return err + } + return VerifyLMConfiguredOnMC(ctx, deployment) +} + +// VerifyMCConfigForCluster verifies that the CM, deployer, search heads, and indexers +// are all correctly registered in the MC config map and pod config string. +// It uses the service name and URL key from the MCVersionConfig so it works for both V3 and V4. +func VerifyMCConfigForCluster(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + cfg MCVersionConfig, mcName string, shPods, indexerPods []string) error { + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(cfg.CMServiceNameFmt, deployment.GetName())}, cfg.CMURLKey, mcName, true); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true); err != nil { + return err + } + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcName, true, false); err != nil { + return err + } + return testcaseEnvInst.VerifyPodsInMCConfigString(ctx, indexerPods, mcName, true, true) +} + +// VerifyStandalonePodsInMC verifies that the given standalone pods are present (or absent) in the +// MC config map and pod config string. +func VerifyStandalonePodsInMC(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, pods []string, mcName string, shouldExist bool) error { + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, pods, "SPLUNK_STANDALONE_URL", mcName, shouldExist); err != nil { + return err + } + return testcaseEnvInst.VerifyPodsInMCConfigString(ctx, pods, mcName, shouldExist, false) +} + +// VerifyMCTwoAfterCMReconfig verifies that MC Two is correctly configured after the Cluster Manager +// has been reconfigured to point to it: CM and indexers should be present, SH should be absent. +// If checkDeployerAbsent is true, also verifies deployer is absent on MC Two (used in C3 tests). +func VerifyMCTwoAfterCMReconfig(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, checkDeployerAbsent bool) error { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, indexerPods, mcTwoName, true, true); err != nil { + return err + } + + if checkDeployerAbsent { + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC Two Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false); err != nil { + return err + } + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config String after CM Reconfig") + return testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcTwoName, false, false) +} + +// VerifyMCOneAfterCMReconfig verifies that MC One is correctly configured after the Cluster Manager +// has been reconfigured away from it: CM should be absent, SH should still be present. +// If checkDeployerPresent is true, also verifies deployer is still present on MC One (used in M4 tests). +func VerifyMCOneAfterCMReconfig(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, checkDeployerPresent bool) error { + + if err := testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false); err != nil { + return err + } + + // CSPL-619: Indexer verification on MC One is commented out in all test variants + + if checkDeployerPresent { + testcaseEnvInst.Log.Info("Verify Deployer still in MC One Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true); err != nil { + return err + } + } + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config Map after CM Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config String after CM Reconfig") + return testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcName, true, false) +} + +// VerifyMCTwoAfterSHCReconfig verifies that MC Two has all components (CM, deployer, SH, indexers) +// after the SHC has been reconfigured to point to it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCTwoAfterSHCReconfig(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, timeout time.Duration) error { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify Deployer in MC Two Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true); err != nil { + return err + } + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig (with wait)") + if err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, shPods, mcTwoName, true, false, timeout); err != nil { + return fmt.Errorf("timed out waiting for search heads in MC two config after SHC reconfig: %w", err) + } + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig (with wait)") + if err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, indexerPods, mcTwoName, true, true, timeout); err != nil { + return fmt.Errorf("timed out waiting for indexers in MC two config after SHC reconfig: %w", err) + } + } else { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcTwoName, true, false); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, indexerPods, mcTwoName, true, true); err != nil { + return err + } + } + return nil +} + +// VerifyMCOneAfterSHCReconfig verifies that MC One has lost all components (CM, deployer, SH) +// after the SHC has been reconfigured away from it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCOneAfterSHCReconfig(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, timeout time.Duration) error { + + if err := testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC One Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false); err != nil { + return err + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config Map after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false); err != nil { + return err + } + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig (with wait)") + if err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, shPods, mcName, false, false, timeout); err != nil { + return fmt.Errorf("timed out waiting for search heads to be removed from MC one config after SHC reconfig: %w", err) + } + } else { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig") + if err := testcaseEnvInst.VerifyPodsInMCConfigString(ctx, shPods, mcName, false, false); err != nil { + return err + } + } + + // CSPL-619: Indexer verification on MC One is commented out in all test variants + return nil +} + +// VerifySecretsPropagated checks that the given secret data has been propagated to all +// versioned secret objects, pods, server config, input config, and via the API. +func VerifySecretsPropagated(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, secretData map[string][]byte, updated bool) error { + // Once Pods are READY check each versioned secret for updated secret keys + secretObjectNames := GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) + + // Verify Secrets on versioned secret objects + if err := testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretData, updated); err != nil { + return err + } + + // Once Pods are READY check each pod for updated secret keys + verificationPods := DumpGetPods(testcaseEnvInst.GetName()) + + // Verify secrets on pods + if err := testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretData, updated); err != nil { + return err + } + + // Verify Secrets on ServerConf on Pod + if err := testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretData, updated); err != nil { + return err + } + + // Verify Hec token on InputConf on Pod + if err := testcaseEnvInst.VerifySplunkInputConfSecrets(verificationPods, secretData, updated); err != nil { + return err + } + + // Verify Secrets via api access on Pod + return testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretData, updated) +} + +// VerifyLMAndStandaloneReady waits for License Manager then Standalone to reach READY status. +func VerifyLMAndStandaloneReady(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig, standalone *enterpriseApi.Standalone) error { + if err := config.LicenseManagerReady(ctx, deployment, testcaseEnvInst); err != nil { + return err + } + return testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) +} + +// VerifyLMAndClusterManagerReady waits for License Manager then Cluster Manager to reach READY status. +func VerifyLMAndClusterManagerReady(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig) error { + if err := config.LicenseManagerReady(ctx, deployment, testcaseEnvInst); err != nil { + return err + } + return config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) +} + +// VerifyS1SecretChangeApplied verifies that a secret change (update or delete) +// has been applied to the S1 stack: standalone enters Updating phase, LM and +// standalone return to Ready, MC version changes, and secrets are propagated. +func VerifyS1SecretChangeApplied(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig, setup S1WithLMSetup, secretData map[string][]byte, updated bool) error { + if err := testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, enterpriseApi.PhaseUpdating); err != nil { + return err + } + if err := VerifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, setup.Standalone); err != nil { + return err + } + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, setup.Mc, setup.ResourceVersion) + return VerifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretData, updated) +} + +// VerifyPostSecretChangeCluster performs the common tail verification after a +// secret change on a clustered deployment: MC version changed, RF/SF met, and +// secrets propagated to all pods. +func VerifyPostSecretChangeCluster(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, mc *enterpriseApi.MonitoringConsole, resourceVersion string, updatedSecretData map[string][]byte) error { + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + testcaseEnvInst.Log.Info("Checking RF SF after secret change") + if err := testcaseEnvInst.VerifyRFSFMet(ctx, deployment); err != nil { + return err + } + + return VerifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} + +// VerifyConfFileContent retrieves a conf file from a pod and validates its content. +func VerifyConfFileContent(pod, confPath, deploymentName string, expectedContent []string, errorMsg string) error { + conf, err := GetConfFile(pod, confPath, deploymentName) + if err != nil { + return fmt.Errorf("%s: %w", errorMsg, err) + } + return ValidateContent(conf, expectedContent, true) +} + +// GetInstanceWithExpect retrieves a CR instance and returns error on failure +func GetInstanceWithExpect(ctx context.Context, deployment *Deployment, instance client.Object, name string, message string) error { + err := deployment.GetInstance(ctx, name, instance) + if err != nil { + return fmt.Errorf("%s: %w", message, err) + } + return nil +} + +// UpdateCRWithExpect updates a CR and returns error on failure +func UpdateCRWithExpect(ctx context.Context, deployment *Deployment, cr client.Object, message string) error { + err := deployment.UpdateCR(ctx, cr) + if err != nil { + return fmt.Errorf("%s: %w", message, err) + } + return nil +} + +// ApplySecretUpdateAndVerifyCMUpdating deploys MC, verifies RF/SF and initial secret state, +// applies a secret update, and confirms the Cluster Manager enters the Updating phase. +// Returns the MC, its resource version, and the updated secret data for post-change verification. +func ApplySecretUpdateAndVerifyCMUpdating(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, config *ClusterReadinessConfig) (*enterpriseApi.MonitoringConsole, string, map[string][]byte, error) { + mc, resourceVersion, err := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + if err != nil { + return nil, "", nil, fmt.Errorf("unable to deploy Monitoring Console: %w", err) + } + testcaseEnvInst.Log.Info("Checking RF SF before secret change") + if err := testcaseEnvInst.VerifyRFSFMet(ctx, deployment); err != nil { + return nil, "", nil, err + } + namespaceScopedSecretName := fmt.Sprintf(NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + _, err = GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + if err != nil { + return nil, "", nil, fmt.Errorf("unable to get secret struct: %w", err) + } + updatedSecretData, err := GenerateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + if err != nil { + return nil, "", nil, fmt.Errorf("unable to generate and apply secret update: %w", err) + } + if err := config.VerifyClusterManagerPhaseUpdating(ctx, deployment, testcaseEnvInst); err != nil { + return nil, "", nil, err + } + return mc, resourceVersion, updatedSecretData, nil +} diff --git a/test/testenv/watch_utils.go b/test/testenv/watch_utils.go index e9b455164..45d3b963e 100644 --- a/test/testenv/watch_utils.go +++ b/test/testenv/watch_utils.go @@ -119,7 +119,7 @@ func (testenv *TestCaseEnv) WatchForIngestorClusterPhase(ctx context.Context, de // WatchForAppPhaseChange uses optimized polling to wait for app phase changes on a CR func (testenv *TestCaseEnv) WatchForAppPhaseChange(ctx context.Context, deployment *Deployment, namespace, crName, crKind, appSourceName, appName string, expectedPhase enterpriseApi.AppPhaseType, timeout time.Duration) error { return wait.PollUntilContextTimeout(ctx, PollInterval, timeout, true, func(ctx context.Context) (bool, error) { - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, crName, crKind, appSourceName, appName) + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, crName, crKind, appSourceName, appName) if err != nil { testenv.Log.Info("Failed to get app deployment info", "app", appName, "error", err) return false, nil @@ -144,7 +144,7 @@ func (testenv *TestCaseEnv) WatchForAllAppsPhaseChange(ctx context.Context, depl } } - appDeploymentInfo, err := GetAppDeploymentInfo(ctx, deployment, testenv, crName, crKind, appSourceName, lookupAppName) + appDeploymentInfo, err := testenv.GetAppDeploymentInfo(ctx, deployment, crName, crKind, appSourceName, lookupAppName) if err != nil { testenv.Log.Info("Failed to get app deployment info", "app", appName, "error", err) return false, nil