diff --git a/Makefile b/Makefile index 93bc5152..81eb2faa 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ test-e2e: build ## Run end-to-end tests (requires kubectl context and cluster ac echo "❌ skopeo not found. Please install skopeo for E2E tests."; \ exit 1; \ fi - $(GOTEST) -v -tags=e2e -timeout=120m ./tests/e2e/... + $(GOTEST) -v -tags=e2e -timeout=120m -parallel=1 ./tests/e2e/... .PHONY: test-all test-all: test test-e2e ## Run all tests (unit + e2e) diff --git a/cmd/deploy.go b/cmd/deploy.go index 479c7871..507a190c 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -30,6 +30,7 @@ Examples: } cmd.Flags().BoolVar(&helm, "helm", false, "Deploy using Helm charts instead of operator") + cmd.Flags().BoolVar(&olm, "olm", false, "Deploy operator via OLM (requires OLM installed)") cmd.Flags().BoolVar(&portForwarding, "port-forwarding", false, "Enable localhost port-forward for Central") cmd.Flags().StringVar(&overrideFile, "override", "", "Path to YAML file with overrides") cmd.Flags().StringArrayVar(&overrideSetExpressions, "set", []string{}, "Set override values (can specify multiple times, e.g., --set foo.bar=val)") @@ -105,12 +106,22 @@ func runDeploy(cmd *cobra.Command, args []string) error { d.SetEnvrcFile(envrc) } + if helm && olm { + return errors.New("cannot use both --helm and --olm flags together") + } + if helm { if err := d.SetUseHelm(true); err != nil { return err } } + if olm { + if err := d.SetUseOLM(true); err != nil { + return err + } + } + d.SetVerbose(verbose) d.SetEarlyReadiness(earlyReadiness) d.SetPortForwardingEnabled(portForwardEnabledFinal) diff --git a/cmd/main.go b/cmd/main.go index 5812e379..64aa6390 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -12,6 +12,7 @@ var ( verbose bool earlyReadiness bool helm bool + olm bool portForwarding bool overrideFile string overrideSetExpressions []string diff --git a/pkg/deployer/deploy_via_operator.go b/pkg/deployer/deploy_via_operator.go index 0807fd8b..f97dbf58 100644 --- a/pkg/deployer/deploy_via_operator.go +++ b/pkg/deployer/deploy_via_operator.go @@ -14,33 +14,76 @@ import ( "gopkg.in/yaml.v3" ) -// deployCentralOperator deploys Central using the operator -func (d *Deployer) deployCentralOperator(ctx context.Context, resources, exposure string) error { - d.logger.Info("πŸš€ Deploying Central via Operator...") - +// ensureOperatorDeployed ensures the operator is deployed with the correct version and mode +func (d *Deployer) ensureOperatorDeployed(ctx context.Context) error { if err := d.ensureCRDsInstalled(ctx); err != nil { return fmt.Errorf("failed to ensure CRDs installed: %w", err) } - operatorDeployed := d.isOperatorDeployed(ctx) - needsDeployment := !operatorDeployed - - if operatorDeployed { - // Operator exists, check if version is correct + // Detect current operator deployment mode + operatorExists, currentMode := d.detectOperatorDeploymentMode(ctx) + needsDeployment := false + needsTeardown := false + + if !operatorExists { + needsDeployment = true + } else if d.useOLM && currentMode == OperatorModeNonOLM { + // Switching from non-OLM to OLM + d.logger.Info("πŸ”„ Switching operator from non-OLM to OLM mode...") + needsTeardown = true + needsDeployment = true + } else if !d.useOLM && currentMode == OperatorModeOLM { + // Switching from OLM to non-OLM + d.logger.Info("πŸ”„ Switching operator from OLM to non-OLM mode...") + needsTeardown = true + needsDeployment = true + } else { + // Same mode, check version if d.isOperatorVersionCorrect(ctx) { d.logger.Info("βœ“ Operator already deployed with correct version") } else { d.logger.Info("πŸ”„ Operator version mismatch, redeploying...") + needsTeardown = true needsDeployment = true } } + if needsTeardown { + // Perform teardown for the current mode + if currentMode == OperatorModeOLM { + if err := d.teardownOperatorOLM(ctx); err != nil { + return fmt.Errorf("failed to teardown OLM operator: %w", err) + } + } else { + if err := d.teardownOperatorNonOLM(ctx); err != nil { + return fmt.Errorf("failed to teardown non-OLM operator: %w", err) + } + } + } + if needsDeployment { - if err := d.deployOperator(ctx); err != nil { - return fmt.Errorf("failed to deploy operator: %w", err) + if d.useOLM { + if err := d.deployOperatorViaOLM(ctx); err != nil { + return fmt.Errorf("failed to deploy operator via OLM: %w", err) + } + } else { + if err := d.deployOperator(ctx); err != nil { + return fmt.Errorf("failed to deploy operator: %w", err) + } } } + return nil +} + +// deployCentralOperator deploys Central using the operator +func (d *Deployer) deployCentralOperator(ctx context.Context, resources, exposure string) error { + d.logger.Info("πŸš€ Deploying Central via Operator...") + + if err := d.ensureOperatorDeployed(ctx); err != nil { + return err + } + if err := d.prepareNamespace(ctx, d.centralNamespace); err != nil { return fmt.Errorf("failed to prepare namespace: %w", err) } @@ -65,14 +108,6 @@ func (d *Deployer) deployCentralOperator(ctx context.Context, resources, exposur return d.configureCentralEndpoint(ctx, exposure) } -// isOperatorDeployed checks if the operator is already deployed -func (d *Deployer) isOperatorDeployed(ctx context.Context) bool { - _, err := d.runKubectl(ctx, KubectlOptions{ - Args: []string{"get", "deployment", operatorDeploymentName, "-n", operatorNamespace}, - }) - return err == nil -} - // isOperatorVersionCorrect checks if the deployed operator matches the desired version func (d *Deployer) isOperatorVersionCorrect(ctx context.Context) bool { currentImage, err := d.getDeployedOperatorImage(ctx) @@ -520,31 +555,12 @@ func (d *Deployer) configureCentralEndpoint(ctx context.Context, exposure string return nil } -// deploySecuredClusterOperator deploys SecuredCluster using the operator +// deploySecuredClusterOperator deploys SecuredCluster using the operator. func (d *Deployer) deploySecuredClusterOperator(ctx context.Context, resources string) error { d.logger.Info("πŸš€ Deploying SecuredCluster via Operator...") - if err := d.ensureCRDsInstalled(ctx); err != nil { - return fmt.Errorf("failed to ensure CRDs installed: %w", err) - } - - operatorDeployed := d.isOperatorDeployed(ctx) - needsDeployment := !operatorDeployed - - if operatorDeployed { - // Operator exists, check if version is correct - if d.isOperatorVersionCorrect(ctx) { - d.logger.Info("βœ“ Operator already deployed with correct version") - } else { - d.logger.Info("πŸ”„ Operator version mismatch, redeploying...") - needsDeployment = true - } - } - - if needsDeployment { - if err := d.deployOperator(ctx); err != nil { - return fmt.Errorf("failed to deploy operator: %w", err) - } + if err := d.ensureOperatorDeployed(ctx); err != nil { + return err } if err := d.prepareNamespace(ctx, d.sensorNamespace); err != nil { @@ -579,7 +595,7 @@ func (d *Deployer) deploySecuredClusterOperator(ctx context.Context, resources s return nil } -// createSecuredClusterCR creates the SecuredCluster custom resource +// createSecuredClusterCR creates the SecuredCluster custom resource. func (d *Deployer) createSecuredClusterCR(clusterName, resources string) (map[string]interface{}, error) { base := map[string]interface{}{ "apiVersion": "platform.stackrox.io/v1alpha1", diff --git a/pkg/deployer/deployer.go b/pkg/deployer/deployer.go index c7b7078c..fb65be77 100644 --- a/pkg/deployer/deployer.go +++ b/pkg/deployer/deployer.go @@ -51,6 +51,7 @@ type Deployer struct { overrideSetExpressions []string envrcFile string useHelm bool + useOLM bool verbose bool earlyReadiness bool } @@ -462,6 +463,11 @@ func (d *Deployer) SetUseHelm(useHelm bool) error { return nil } +func (d *Deployer) SetUseOLM(useOLM bool) error { + d.useOLM = useOLM + return nil +} + func (d *Deployer) SetVerbose(verbose bool) { d.verbose = verbose } @@ -571,6 +577,7 @@ func (d *Deployer) PrintCentralDeploymentSummary() { component := "Central" mainImageTag := d.mainImageTag helm := d.useHelm + olm := d.useOLM exposure := d.exposure portForwarding := d.portForwardEnabled log := d.logger @@ -627,6 +634,11 @@ func (d *Deployer) PrintCentralDeploymentSummary() { log.Info(cyan.Sprint("β”‚") + createRow("Main Tag", mainImageTag)) log.Info(cyan.Sprint("β”‚") + createRow("Kubernetes Context", kubeContext)) log.Info(cyan.Sprint("β”‚") + createRow("Deployment Method", map[bool]string{true: "Helm", false: "Operator"}[helm])) + + if olm { + log.Info(cyan.Sprint("β”‚") + createRow("OLM", "Yes")) + } + log.Info(cyan.Sprint("β”‚") + createRow("Exposure", exposure)) if portForwarding || exposure == "none" { @@ -734,6 +746,7 @@ func (d *Deployer) PrintSecuredClusterDeploymentSummary() { component := "Secured Cluster" mainImageTag := d.mainImageTag helm := d.useHelm + olm := d.useOLM log := d.logger kubeContext := d.kubeContext @@ -789,6 +802,10 @@ func (d *Deployer) PrintSecuredClusterDeploymentSummary() { log.Info(cyan.Sprint("β”‚") + createRow("Kubernetes Context", kubeContext)) log.Info(cyan.Sprint("β”‚") + createRow("Deployment Method", map[bool]string{true: "Helm", false: "Operator"}[helm])) + if olm { + log.Info(cyan.Sprint("β”‚") + createRow("OLM", "Yes")) + } + log.Info(cyan.Sprint("β””" + strings.Repeat("─", boxWidth) + "β”˜")) log.Info("") } diff --git a/pkg/deployer/operator.go b/pkg/deployer/operator.go index b5e125a9..eaa14f32 100644 --- a/pkg/deployer/operator.go +++ b/pkg/deployer/operator.go @@ -514,3 +514,36 @@ func generateClusterName() string { n, _ := rand.Int(rand.Reader, big.NewInt(9000)) return fmt.Sprintf("sensor-%d", n.Int64()+1000) } + +// teardownOperatorNonOLM removes the operator when installed without OLM. +func (d *Deployer) teardownOperatorNonOLM(ctx context.Context) error { + d.logger.Info("🧹 Tearing down operator deployed without OLM...") + + // Delete operator namespace. + d.runKubectl(ctx, KubectlOptions{ + Args: []string{"delete", "namespace", operatorNamespace, "--wait=false"}, + IgnoreErrors: true, + }) + + // Delete cluster-scoped resources created by non-OLM flow. + clusterResources := []struct { + name string + kind string + }{ + {name: "rhacs-operator-manager-rolebinding", kind: "clusterrolebinding"}, + {name: "rhacs-operator-manager-role", kind: "clusterrole"}, + } + for _, resource := range clusterResources { + d.runKubectl(ctx, KubectlOptions{ + Args: []string{"delete", resource.kind, resource.name, "--ignore-not-found=true"}, + IgnoreErrors: true, + }) + } + + if err := d.waitForNamespaceDeletion(operatorNamespace); err != nil { + d.logger.Warningf("Namespace %s deletion incomplete: %v", operatorNamespace, err) + } + + d.logger.Success("βœ“ Non-OLM operator resources removed") + return nil +} diff --git a/pkg/deployer/operator_olm.go b/pkg/deployer/operator_olm.go new file mode 100644 index 00000000..ceea9679 --- /dev/null +++ b/pkg/deployer/operator_olm.go @@ -0,0 +1,413 @@ +package deployer + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + catalogSourceName = "stackrox-operator-index" + subscriptionName = "stackrox-operator-subscription" + operatorGroupName = "all-namespaces-operator-group" + operatorChannel = "latest" +) + +// OperatorDeploymentMode represents how the operator is deployed +type OperatorDeploymentMode bool + +const ( + OperatorModeNonOLM OperatorDeploymentMode = false + OperatorModeOLM OperatorDeploymentMode = true +) + +// deployOperatorViaOLM deploys the RHACS operator using OLM. +func (d *Deployer) deployOperatorViaOLM(ctx context.Context) error { + d.logger.Info("πŸš€ Deploying operator via OLM...") + d.logger.Infof("Operator tag: %s", d.operatorTag) + + // Sanity check:Check if OLM is installed. + if err := d.checkOLMInstalled(ctx); err != nil { + return err + } + + indexImage := d.getOperatorIndexImage() + d.logger.Infof("Index image: %s", indexImage) + + // Create operator namespace. + if err := d.createOperatorNamespace(ctx); err != nil { + return err + } + + // Create CatalogSource. + if err := d.createCatalogSource(ctx, indexImage); err != nil { + return fmt.Errorf("failed to create CatalogSource: %w", err) + } + + // Create OperatorGroup. + if err := d.createOperatorGroup(ctx); err != nil { + return fmt.Errorf("failed to create OperatorGroup: %w", err) + } + + // Create Subscription. + if err := d.createSubscription(ctx); err != nil { + return fmt.Errorf("failed to create Subscription: %w", err) + } + + // Wait for and approve InstallPlan. + if err := d.waitForAndApproveInstallPlan(ctx); err != nil { + return fmt.Errorf("failed to approve InstallPlan: %w", err) + } + + // Wait for CSV to succeed. + if err := d.waitForCSVSuccess(ctx); err != nil { + return fmt.Errorf("failed waiting for CSV: %w", err) + } + + // Wait for operator deployment. + if err := d.waitForOperatorReady(ctx, operatorNamespace, operatorDeploymentName, 300); err != nil { + return fmt.Errorf("failed waiting for operator: %w", err) + } + + d.logger.Success("πŸŽ‰ Operator deployed successfully via OLM!") + return nil +} + +// checkOLMInstalled checks if OLM is installed in the cluster. +func (d *Deployer) checkOLMInstalled(ctx context.Context) error { + // Check for OLM CRDs + requiredCRDs := []string{ + "catalogsources.operators.coreos.com", + "subscriptions.operators.coreos.com", + "installplans.operators.coreos.com", + "clusterserviceversions.operators.coreos.com", + } + + for _, crd := range requiredCRDs { + _, err := d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "crd", crd}, + }) + if err != nil { + return fmt.Errorf("OLM not installed: CRD %s not found. Please install OLM first", crd) + } + } + + d.logger.Success("βœ“ OLM detected in cluster") + return nil +} + +// getOperatorIndexImage returns the operator index image reference. +func (d *Deployer) getOperatorIndexImage() string { + return fmt.Sprintf("quay.io/rhacs-eng/stackrox-operator-index:v%s", d.operatorTag) +} + +// createCatalogSource creates the CatalogSource for the operator. +func (d *Deployer) createCatalogSource(ctx context.Context, indexImage string) error { + d.logger.Info("Creating CatalogSource...") + + // Check if CatalogSource CRD supports securityContextConfig. + hasSecurityContextConfig, err := d.catalogSourceSupportsSecurityContextConfig(ctx) + if err != nil { + d.logger.Warning("Could not check CatalogSource CRD capabilities, proceeding without securityContextConfig") + hasSecurityContextConfig = false + } + + catalogSource := map[string]interface{}{ + "apiVersion": "operators.coreos.com/v1alpha1", + "kind": "CatalogSource", + "metadata": map[string]interface{}{ + "name": catalogSourceName, + "namespace": operatorNamespace, + }, + "spec": map[string]interface{}{ + "sourceType": "grpc", + "image": indexImage, + "displayName": "StackRox Operator Index", + }, + } + + // Add security context config if supported (OCP 4.14+). + if hasSecurityContextConfig { + spec := catalogSource["spec"].(map[string]interface{}) + spec["grpcPodConfig"] = map[string]interface{}{ + "securityContextConfig": "restricted", + } + } + + yamlData, err := yaml.Marshal(catalogSource) + if err != nil { + return fmt.Errorf("failed to marshal CatalogSource: %w", err) + } + + _, err = d.runKubectl(ctx, KubectlOptions{ + Args: []string{"apply", "-f", "-"}, + Stdin: bytes.NewReader(yamlData), + }) + if err != nil { + return fmt.Errorf("failed to create CatalogSource: %w", err) + } + + d.logger.Success("βœ“ CatalogSource created") + return nil +} + +// catalogSourceSupportsSecurityContextConfig checks if the CatalogSource CRD supports securityContextConfig. +func (d *Deployer) catalogSourceSupportsSecurityContextConfig(ctx context.Context) (bool, error) { + result, err := d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "crd", "catalogsources.operators.coreos.com", "-o", "yaml"}, + }) + if err != nil { + return false, err + } + + return strings.Contains(result.Stdout, "securityContextConfig"), nil +} + +// createOperatorGroup creates the OperatorGroup. +func (d *Deployer) createOperatorGroup(ctx context.Context) error { + d.logger.Info("Creating OperatorGroup...") + + operatorGroup := map[string]interface{}{ + "apiVersion": "operators.coreos.com/v1alpha2", + "kind": "OperatorGroup", + "metadata": map[string]interface{}{ + "name": operatorGroupName, + "namespace": operatorNamespace, + }, + } + + yamlData, err := yaml.Marshal(operatorGroup) + if err != nil { + return fmt.Errorf("failed to marshal OperatorGroup: %w", err) + } + + _, err = d.runKubectl(ctx, KubectlOptions{ + Args: []string{"apply", "-f", "-"}, + Stdin: bytes.NewReader(yamlData), + }) + if err != nil { + return fmt.Errorf("failed to create OperatorGroup: %w", err) + } + + d.logger.Success("βœ“ OperatorGroup created") + return nil +} + +// createSubscription creates the Subscription for the operator. +func (d *Deployer) createSubscription(ctx context.Context) error { + d.logger.Info("Creating Subscription...") + + startingCSV := fmt.Sprintf("rhacs-operator.v%s", d.operatorTag) + + subscription := map[string]interface{}{ + "apiVersion": "operators.coreos.com/v1alpha1", + "kind": "Subscription", + "metadata": map[string]interface{}{ + "name": subscriptionName, + "namespace": operatorNamespace, + }, + "spec": map[string]interface{}{ + "channel": operatorChannel, + "name": "rhacs-operator", + "source": catalogSourceName, + "sourceNamespace": operatorNamespace, + "installPlanApproval": "Manual", + "startingCSV": startingCSV, + }, + } + + yamlData, err := yaml.Marshal(subscription) + if err != nil { + return fmt.Errorf("failed to marshal Subscription: %w", err) + } + + _, err = d.runKubectl(ctx, KubectlOptions{ + Args: []string{"apply", "-f", "-"}, + Stdin: bytes.NewReader(yamlData), + }) + if err != nil { + return fmt.Errorf("failed to create Subscription: %w", err) + } + + d.logger.Success("βœ“ Subscription created") + return nil +} + +// waitForAndApproveInstallPlan waits for the InstallPlan to be created and approves it. +func (d *Deployer) waitForAndApproveInstallPlan(ctx context.Context) error { + d.logger.Info("⏳ Waiting for InstallPlan to be created...") + + // Wait for subscription to have InstallPlanPending condition. + start := time.Now() + timeout := 5 * time.Minute + + for time.Since(start) < timeout { + result, err := d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "subscription", subscriptionName, "-n", operatorNamespace, "-o", "jsonpath={.status.conditions[?(@.type=='InstallPlanPending')].status}"}, + }) + if err == nil && strings.TrimSpace(result.Stdout) == "True" { + break + } + + time.Sleep(5 * time.Second) + } + + if time.Since(start) >= timeout { + return errors.New("timeout waiting for InstallPlan to be created") + } + + // Sanity check:Verify currentCSV matches expected version. + expectedCSV := fmt.Sprintf("rhacs-operator.v%s", d.operatorTag) + result, err := d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "subscription", subscriptionName, "-n", operatorNamespace, "-o", "jsonpath={.status.currentCSV}"}, + }) + if err != nil { + return fmt.Errorf("failed to get current CSV from subscription: %w", err) + } + + currentCSV := strings.TrimSpace(result.Stdout) + if currentCSV != expectedCSV { + return fmt.Errorf("subscription progressing to unexpected CSV '%s', expected '%s'", currentCSV, expectedCSV) + } + + // Get InstallPlan name. + result, err = d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "subscription", subscriptionName, "-n", operatorNamespace, "-o", "jsonpath={.status.installPlanRef.name}"}, + }) + if err != nil { + return fmt.Errorf("failed to get InstallPlan name: %w", err) + } + + installPlanName := strings.TrimSpace(result.Stdout) + if installPlanName == "" { + return errors.New("InstallPlan name is empty") + } + + d.logger.Infof("Approving InstallPlan: %s", installPlanName) + + // Approve the InstallPlan. + _, err = d.runKubectl(ctx, KubectlOptions{ + Args: []string{"patch", "installplan", installPlanName, "-n", operatorNamespace, "--type", "merge", "-p", `{"spec":{"approved":true}}`}, + }) + if err != nil { + return fmt.Errorf("failed to approve InstallPlan: %w", err) + } + + d.logger.Success("βœ“ InstallPlan approved") + return nil +} + +// waitForCSVSuccess waits for the CSV to reach Succeeded phase. +func (d *Deployer) waitForCSVSuccess(ctx context.Context) error { + csvName := fmt.Sprintf("rhacs-operator.v%s", d.operatorTag) + d.logger.Infof("⏳ Waiting for CSV %s to succeed...", csvName) + + start := time.Now() + timeout := 10 * time.Minute + + for time.Since(start) < timeout { + result, err := d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "csv", csvName, "-n", operatorNamespace, "-o", "jsonpath={.status.phase}"}, + }) + if err == nil { + phase := strings.TrimSpace(result.Stdout) + if phase == "Succeeded" { + d.logger.Success("βœ“ CSV succeeded") + return nil + } + if phase == "Failed" { + return fmt.Errorf("CSV entered Failed phase") + } + } + + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for CSV to succeed") +} + +// detectOperatorDeploymentMode detects how the operator is currently deployed. +// Returns (operatorExists bool, isOLM OperatorDeploymentMode) +func (d *Deployer) detectOperatorDeploymentMode(ctx context.Context) (bool, OperatorDeploymentMode) { + // First, check if a Subscription exists (OLM-specific resource) + _, err := d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "subscription", subscriptionName, "-n", operatorNamespace}, + }) + if err == nil { + return true, OperatorModeOLM + } + + // If no subscription, check if operator deployment exists. + _, err = d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "deployment", operatorDeploymentName, "-n", operatorNamespace}, + }) + if err == nil { + // Deployment exists - check if it has OLM owner labels. + result, err := d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "deployment", operatorDeploymentName, "-n", operatorNamespace, "-o", "jsonpath={.metadata.labels}"}, + }) + if err == nil && strings.Contains(result.Stdout, "olm.owner") { + return true, OperatorModeOLM + } + // Deployment exists without OLM labels = non-OLM deployment. + return true, OperatorModeNonOLM + } + + // No operator found. + return false, OperatorModeNonOLM +} + +// teardownOperatorOLM removes the operator when installed via OLM. +func (d *Deployer) teardownOperatorOLM(ctx context.Context) error { + d.logger.Info("🧹 Tearing down operator deployed via OLM...") + + // Delete Subscription (this typically cascades CSV and operands depending on OLM behavior). + d.runKubectl(ctx, KubectlOptions{ + Args: []string{"delete", "subscription", subscriptionName, "-n", operatorNamespace, "--ignore-not-found=true"}, + IgnoreErrors: true, + }) + + // Find the CSV name (may match operatorTag, but query to be safe). + result, err := d.runKubectl(ctx, KubectlOptions{ + Args: []string{"get", "csv", "-n", operatorNamespace, "-o", "jsonpath={.items[*].metadata.name}"}, + }) + if err == nil { + // Best-effort delete all matching CSVs for rhacs-operator. + for _, name := range strings.Fields(strings.TrimSpace(result.Stdout)) { + if strings.HasPrefix(name, "rhacs-operator.v") { + d.runKubectl(ctx, KubectlOptions{ + Args: []string{"delete", "csv", name, "-n", operatorNamespace, "--ignore-not-found=true"}, + IgnoreErrors: true, + }) + } + } + } + + // Delete CatalogSource and OperatorGroup. + d.runKubectl(ctx, KubectlOptions{ + Args: []string{"delete", "catalogsource", catalogSourceName, "-n", operatorNamespace, "--ignore-not-found=true"}, + IgnoreErrors: true, + }) + d.runKubectl(ctx, KubectlOptions{ + Args: []string{"delete", "operatorgroup", operatorGroupName, "-n", operatorNamespace, "--ignore-not-found=true"}, + IgnoreErrors: true, + }) + + // Delete operator deployment namespace (contains deployment, SA, etc.). + d.runKubectl(ctx, KubectlOptions{ + Args: []string{"delete", "namespace", operatorNamespace, "--ignore-not-found=true", "--wait=false"}, + IgnoreErrors: true, + }) + + if err := d.waitForNamespaceDeletion(operatorNamespace); err != nil { + d.logger.Warningf("Namespace %s deletion incomplete: %v", operatorNamespace, err) + } + + d.logger.Success("βœ“ OLM operator resources removed") + return nil +} diff --git a/tests/README.md b/tests/README.md index 01b29728..4768b5ac 100644 --- a/tests/README.md +++ b/tests/README.md @@ -41,13 +41,15 @@ make test-e2e Or directly with go: ```bash -go test -v -tags=e2e -timeout=30m ./tests/e2e/... +# Note: -parallel=1 ensures tests run sequentially to avoid conflicts +go test -v -tags=e2e -timeout=30m -parallel=1 ./tests/e2e/... ``` ### Environment Variables for E2E Tests - `MAIN_IMAGE_TAG` - ACS image tag to use (default: "4.8.2") - `SKIP_OPERATOR_TESTS` - Skip operator-based tests if set +- `SKIP_OLM_TESTS` - Skip OLM-specific tests if set (useful when OLM is not available) - `SKIP_IMAGE_VERIFICATION` - Skip image verification if set to "true" Example: @@ -55,6 +57,27 @@ Example: MAIN_IMAGE_TAG=4.9.0 make test-e2e ``` +### Running OLM Switching Tests + +The OLM switching tests verify that roxie can properly switch between OLM and non-OLM operator deployment modes. These tests: +- Require OLM installed in the cluster +- Require sufficient cluster resources +- Test all switching scenarios: OLM↔non-OLM, version upgrades, multi-component deployments + +Run OLM tests: +```bash +# Run all OLM tests (they run sequentially via -parallel=1) +go test -v -tags=e2e -timeout=120m -parallel=1 ./tests/e2e/ -run TestOLM + +# Run specific OLM test +go test -v -tags=e2e -timeout=60m -parallel=1 ./tests/e2e/ -run TestOLMToNonOLMSwitch +``` + +Skip OLM tests if OLM is not available: +```bash +SKIP_OLM_TESTS=1 go test -v -tags=e2e -timeout=30m -parallel=1 ./tests/e2e/ +``` + ## Test Organization ### Unit Tests @@ -108,10 +131,14 @@ func TestMyFunction(t *testing.T) { package e2e import ( + "os" "testing" "time" ) +// Note: All e2e tests run sequentially via the -parallel=1 flag in the Makefile +// to prevent conflicts when modifying shared cluster resources. + func TestDeployment(t *testing.T) { if os.Getenv("SKIP_OPERATOR_TESTS") != "" { t.Skip("Operator tests disabled") diff --git a/tests/e2e/olm_switch_test.go b/tests/e2e/olm_switch_test.go new file mode 100644 index 00000000..155f0f38 --- /dev/null +++ b/tests/e2e/olm_switch_test.go @@ -0,0 +1,291 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// Note: All e2e tests run sequentially via the -parallel=1 flag in the Makefile. +// This prevents conflicts when multiple tests modify shared cluster resources. + +// verifyOperatorMode checks if the operator is deployed in the expected mode (OLM or non-OLM) +func verifyOperatorMode(t *testing.T, expectOLM bool) { + t.Helper() + + operatorNamespace := "rhacs-operator-system" + subscriptionName := "stackrox-operator-subscription" + + // Check for OLM Subscription (OLM-specific resource) + cmd := exec.Command("kubectl", "get", "subscription", subscriptionName, "-n", operatorNamespace) + err := cmd.Run() + olmExists := (err == nil) + + if expectOLM && !olmExists { + t.Fatalf("Expected OLM operator but Subscription not found") + } + if !expectOLM && olmExists { + t.Fatalf("Expected non-OLM operator but Subscription found (indicates OLM deployment)") + } + + t.Logf("βœ“ Operator is deployed in expected mode (OLM: %v)", expectOLM) +} + +// verifyOperatorDeploymentExists checks if the operator deployment exists +func verifyOperatorDeploymentExists(t *testing.T) { + t.Helper() + + operatorNamespace := "rhacs-operator-system" + deploymentName := "rhacs-operator-controller-manager" + + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", operatorNamespace) + if err := cmd.Run(); err != nil { + t.Fatalf("Operator deployment not found: %v", err) + } + + // Also check if deployment is ready + cmd = exec.Command("kubectl", "get", "deployment", deploymentName, "-n", operatorNamespace, + "-o", "jsonpath={.status.readyReplicas}") + output, err := cmd.Output() + if err != nil { + t.Fatalf("Failed to check operator deployment readiness: %v", err) + } + + readyReplicas := strings.TrimSpace(string(output)) + if readyReplicas == "" || readyReplicas == "0" { + t.Fatalf("Operator deployment exists but has no ready replicas") + } + + t.Logf("βœ“ Operator deployment exists and is ready (%s replicas)", readyReplicas) +} + +// TestOLMToNonOLMSwitch tests switching from OLM operator to non-OLM operator +func TestOLMToNonOLMSwitch(t *testing.T) { + if os.Getenv("SKIP_OLM_TESTS") != "" { + t.Skip("SKIP_OLM_TESTS is set") + } + + // Create temporary envrc file + envrcFile, err := os.CreateTemp("", ".envrc.roxie-test-*") + if err != nil { + t.Fatalf("Failed to create temp envrc: %v", err) + } + envrcPath := envrcFile.Name() + envrcFile.Close() + defer os.Remove(envrcPath) + + // Step 1: Deploy central with OLM operator + t.Log("=== Step 1: Deploy central with OLM operator ===") + args := append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, nil, args...) + + // Verify operator is in OLM mode + t.Log("Verifying operator is in OLM mode") + verifyOperatorMode(t, true) + verifyOperatorDeploymentExists(t) + + // Verify central namespace exists + verifyNamespaceExists(t, "acs-central") + + // Brief pause + time.Sleep(10 * time.Second) + + // Step 2: Deploy central again without OLM (should switch modes) + t.Log("=== Step 2: Redeploy central without OLM (triggering mode switch) ===") + args = append([]string{roxieBinary, "deploy", "central", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, nil, args...) + + // Verify operator switched to non-OLM mode + t.Log("Verifying operator switched to non-OLM mode") + verifyOperatorMode(t, false) + verifyOperatorDeploymentExists(t) + + // Verify central namespace still exists + verifyNamespaceExists(t, "acs-central") + + // Cleanup + t.Log("=== Cleaning up ===") + teardownArgs := []string{roxieBinary, "teardown", "central"} + runCommand(t, teardownTimeout, nil, teardownArgs...) + + verifyNamespaceAbsent(t, "acs-central") +} + +// TestNonOLMToOLMSwitch tests switching from non-OLM operator to OLM operator +func TestNonOLMToOLMSwitch(t *testing.T) { + if os.Getenv("SKIP_OLM_TESTS") != "" { + t.Skip("SKIP_OLM_TESTS is set") + } + + // Create temporary envrc file + envrcFile, err := os.CreateTemp("", ".envrc.roxie-test-*") + if err != nil { + t.Fatalf("Failed to create temp envrc: %v", err) + } + envrcPath := envrcFile.Name() + envrcFile.Close() + defer os.Remove(envrcPath) + + // Step 1: Deploy central without OLM (non-OLM operator) + t.Log("=== Step 1: Deploy central with non-OLM operator ===") + args := append([]string{roxieBinary, "deploy", "central", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, nil, args...) + + // Verify operator is in non-OLM mode + t.Log("Verifying operator is in non-OLM mode") + verifyOperatorMode(t, false) + verifyOperatorDeploymentExists(t) + + // Verify central namespace exists + verifyNamespaceExists(t, "acs-central") + + // Brief pause + time.Sleep(10 * time.Second) + + // Step 2: Deploy central again with OLM (should switch modes) + t.Log("=== Step 2: Redeploy central with OLM (triggering mode switch) ===") + args = append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, nil, args...) + + // Verify operator switched to OLM mode + t.Log("Verifying operator switched to OLM mode") + verifyOperatorMode(t, true) + verifyOperatorDeploymentExists(t) + + // Verify central namespace still exists + verifyNamespaceExists(t, "acs-central") + + // Cleanup + t.Log("=== Cleaning up ===") + teardownArgs := []string{roxieBinary, "teardown", "central"} + runCommand(t, teardownTimeout, nil, teardownArgs...) + + verifyNamespaceAbsent(t, "acs-central") +} + +// TestOLMOperatorVersionUpgrade tests that OLM operator version mismatches trigger teardown and redeploy +func TestOLMOperatorVersionUpgrade(t *testing.T) { + if os.Getenv("SKIP_OLM_TESTS") != "" { + t.Skip("SKIP_OLM_TESTS is set") + } + + // This test requires two different operator versions + // We can simulate this by deploying twice with the same version, + // but the logic should handle version changes by tearing down and redeploying + + // Create temporary envrc file + envrcFile, err := os.CreateTemp("", ".envrc.roxie-test-*") + if err != nil { + t.Fatalf("Failed to create temp envrc: %v", err) + } + envrcPath := envrcFile.Name() + envrcFile.Close() + defer os.Remove(envrcPath) + + // Step 1: Deploy central with OLM operator + t.Log("=== Step 1: Deploy central with OLM operator ===") + args := append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, nil, args...) + + // Verify operator is in OLM mode + t.Log("Verifying initial OLM operator deployment") + verifyOperatorMode(t, true) + verifyOperatorDeploymentExists(t) + + // Get the current operator version + operatorNamespace := "rhacs-operator-system" + deploymentName := "rhacs-operator-controller-manager" + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", operatorNamespace, + "-o", "jsonpath={.spec.template.spec.containers[0].image}") + output, err := cmd.Output() + if err != nil { + t.Fatalf("Failed to get operator image: %v", err) + } + initialImage := strings.TrimSpace(string(output)) + t.Logf("Initial operator image: %s", initialImage) + + // Brief pause + time.Sleep(10 * time.Second) + + // Step 2: Redeploy with same version (should skip if version matches) + t.Log("=== Step 2: Redeploy with same version (should detect correct version) ===") + args = append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, nil, args...) + + // Verify operator is still in OLM mode and deployment exists + t.Log("Verifying operator is still deployed correctly") + verifyOperatorMode(t, true) + verifyOperatorDeploymentExists(t) + + // Note: In a real version upgrade test, we would set a different MAIN_IMAGE_TAG + // and verify that the operator was torn down and redeployed with the new version. + // For now, this test validates the basic flow. + + // Cleanup + t.Log("=== Cleaning up ===") + teardownArgs := []string{roxieBinary, "teardown", "central"} + runCommand(t, teardownTimeout, nil, teardownArgs...) + + verifyNamespaceAbsent(t, "acs-central") +} + +// TestSecuredClusterWithOLMSwitch tests that secured-cluster deployment also respects OLM mode switches +func TestSecuredClusterWithOLMSwitch(t *testing.T) { + if os.Getenv("SKIP_OLM_TESTS") != "" { + t.Skip("SKIP_OLM_TESTS is set") + } + + // Create temporary envrc file + envrcFile, err := os.CreateTemp("", ".envrc.roxie-test-*") + if err != nil { + t.Fatalf("Failed to create temp envrc: %v", err) + } + envrcPath := envrcFile.Name() + envrcFile.Close() + defer os.Remove(envrcPath) + + // Step 1: Deploy central with OLM + t.Log("=== Step 1: Deploy central with OLM ===") + args := append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, nil, args...) + + verifyOperatorMode(t, true) + verifyNamespaceExists(t, "acs-central") + + // Load envrc for secured-cluster + envrcEnv, err := loadEnvrcFile(envrcPath) + if err != nil { + t.Fatalf("Failed to load envrc file: %v", err) + } + + // Step 2: Deploy secured-cluster (should reuse OLM operator) + t.Log("=== Step 2: Deploy secured-cluster (should reuse OLM operator) ===") + args = append([]string{roxieBinary, "deploy", "secured-cluster", "--olm"}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, envrcEnv, args...) + + // Verify operator is still in OLM mode + verifyOperatorMode(t, true) + verifyNamespaceExists(t, "acs-sensor") + + // Step 3: Switch to non-OLM by redeploying secured-cluster without --olm + t.Log("=== Step 3: Redeploy secured-cluster without OLM (triggering mode switch) ===") + args = append([]string{roxieBinary, "deploy", "secured-cluster"}, commonDeployArgsNoPortForward...) + runCommand(t, deployTimeout, envrcEnv, args...) + + // Verify operator switched to non-OLM mode + verifyOperatorMode(t, false) + verifyNamespaceExists(t, "acs-sensor") + + // Cleanup + t.Log("=== Cleaning up ===") + teardownArgs := []string{roxieBinary, "teardown", "both"} + runCommand(t, teardownTimeout, nil, teardownArgs...) + + verifyNamespaceAbsent(t, "acs-central") + verifyNamespaceAbsent(t, "acs-sensor") +}