Fix: don't delete Central resources during SecuredCluster teardown (#110)

Co-authored-by: Moritz Clasmeier <mclasmeier@redhat.com>

Author: mclasmeier

internal/deployer/constants.go

@@ -3,6 +3,9 @@ package deployer
 import "fmt"
 
 var (
+	centralCrName        = "stackrox-central-services"
+	securedClusterCrName = "stackrox-secured-cluster-services"
+
 	centralDbPVCSizeSmall = "30Gi"
 
 	centralResourcesSmall = map[string]interface{}{

internal/deployer/crs.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+
+	"github.com/stackrox/roxie/internal/k8s"
 )
 
 // generateCRS generates the Central Resource Secret using roxctl
@@ -43,7 +45,7 @@ func (d *Deployer) generateCRS(ctx context.Context, clusterName string) (string,
 func (d *Deployer) applyCRS(ctx context.Context, crsContent string) error {
 	d.logger.Info("Applying CRS to sensor namespace")
 
-	result, err := d.runKubectl(ctx, KubectlOptions{
+	result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args:  []string{"apply", "-n", d.sensorNamespace, "-f", "-"},
 		Stdin: strings.NewReader(crsContent),
 	})

internal/deployer/deploy_via_operator.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/stackrox/roxie/internal/env"
 	"github.com/stackrox/roxie/internal/helpers"
+	"github.com/stackrox/roxie/internal/k8s"
 	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
@@ -167,7 +168,7 @@ func (d *Deployer) isOperatorVersionCorrect(ctx context.Context) bool {
 
 // getDeployedOperatorImage gets the image of the currently deployed operator
 func (d *Deployer) getDeployedOperatorImage(ctx context.Context) (string, error) {
-	result, err := d.runKubectl(ctx, KubectlOptions{
+	result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args: []string{"get", "deployment", operatorDeploymentName, "-n", operatorNamespace,
 			"-o", "jsonpath={.spec.template.spec.containers[0].image}"},
 	})
@@ -200,7 +201,7 @@ func (d *Deployer) ensurePullSecretExists(ctx context.Context, namespace string)
 	// Assemble pull secret YAML from pre-verified credentials
 	pullSecretYAML := d.dockerAuth.CreatePullSecretYAMLFromCredentials(d.dockerCreds, namespace)
 
-	_, err := d.runKubectl(ctx, KubectlOptions{
+	_, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args:  []string{"apply", "-f", "-"},
 		Stdin: strings.NewReader(pullSecretYAML),
 	})
@@ -231,7 +232,7 @@ func (d *Deployer) createAdminPasswordSecret(ctx context.Context) error {
 		return fmt.Errorf("failed to marshal secret: %w", err)
 	}
 
-	_, err = d.runKubectl(ctx, KubectlOptions{
+	_, err = d.runKubectl(ctx, k8s.KubectlOptions{
 		Args:  []string{"apply", "-f", "-"},
 		Stdin: bytes.NewReader(yamlData),
 	})
@@ -438,7 +439,7 @@ func (d *Deployer) applyCentralCR(ctx context.Context, cr map[string]interface{}
 		}
 	}
 
-	result, err := d.runKubectl(ctx, KubectlOptions{
+	result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args:  []string{"apply", "-f", "-"},
 		Stdin: bytes.NewReader(yamlData),
 	})
@@ -473,7 +474,7 @@ func (d *Deployer) waitForCentralReady(ctx context.Context, timeout time.Duratio
 		}
 
 		// Check if central deployment is ready
-		result, err := d.runKubectl(ctx, KubectlOptions{
+		result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 			Args: []string{"get", "deployment", "central", "-n", d.centralNamespace, "-o", "jsonpath={.status.readyReplicas}"},
 		})
 		if err == nil && result.Stdout != "" {
@@ -508,7 +509,7 @@ func (d *Deployer) waitForLoadBalancer(ctx context.Context, namespace, serviceNa
 
 	start := time.Now()
 	for time.Since(start) < time.Duration(timeout)*time.Second {
-		result, err := d.runKubectl(ctx, KubectlOptions{
+		result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 			Args: []string{"get", "svc", serviceName, "-n", namespace, "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"},
 		})
 		if err == nil && result.Stdout != "" {
@@ -524,7 +525,7 @@ func (d *Deployer) waitForLoadBalancer(ctx context.Context, namespace, serviceNa
 		}
 
 		// Also check for hostname (some cloud providers use hostname instead of IP)
-		result, err = d.runKubectl(ctx, KubectlOptions{
+		result, err = d.runKubectl(ctx, k8s.KubectlOptions{
 			Args: []string{"get", "svc", serviceName, "-n", namespace, "-o", "jsonpath={.status.loadBalancer.ingress[0].hostname}"},
 		})
 		if err == nil && result.Stdout != "" {
@@ -549,7 +550,7 @@ func (d *Deployer) waitForLoadBalancer(ctx context.Context, namespace, serviceNa
 func (d *Deployer) fetchCentralCACert(ctx context.Context) error {
 	d.logger.Info("Fetching Central CA certificate...")
 
-	result, err := d.runKubectl(ctx, KubectlOptions{
+	result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args: []string{"get", "secret", "central-tls", "-n", d.centralNamespace, "-o", "jsonpath={.data.ca\\.pem}"},
 	})
 	if err != nil {
@@ -771,7 +772,7 @@ func (d *Deployer) applySecuredClusterCR(ctx context.Context, cr map[string]inte
 		}
 	}
 
-	result, err := d.runKubectl(ctx, KubectlOptions{
+	result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args:  []string{"apply", "-n", d.sensorNamespace, "-f", "-"},
 		Stdin: bytes.NewReader(yamlData),
 	})
@@ -805,7 +806,7 @@ func (d *Deployer) waitForSecuredClusterReady(ctx context.Context, timeout time.
 		allReady := true
 
 		// Check sensor deployment
-		result, err := d.runKubectl(ctx, KubectlOptions{
+		result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 			Args: []string{"get", "deployment", "sensor", "-n", d.sensorNamespace, "-o", "jsonpath={.status.readyReplicas}"},
 		})
 		if err != nil || result.Stdout == "" {
@@ -820,7 +821,7 @@ func (d *Deployer) waitForSecuredClusterReady(ctx context.Context, timeout time.
 		// Only check additional workloads if early-readiness is not enabled
 		if !d.earlyReadiness {
 			// Check admission-control deployment
-			result, err = d.runKubectl(ctx, KubectlOptions{
+			result, err = d.runKubectl(ctx, k8s.KubectlOptions{
 				Args: []string{"get", "deployment", "admission-control", "-n", d.sensorNamespace, "-o", "jsonpath={.status.readyReplicas}"},
 			})
 			if err != nil || result.Stdout == "" {

internal/deployer/deployer.go

@@ -20,6 +20,7 @@ import (
 	"github.com/stackrox/roxie/internal/env"
 	"github.com/stackrox/roxie/internal/helpers"
 	"github.com/stackrox/roxie/internal/imagecache"
+	"github.com/stackrox/roxie/internal/k8s"
 	"github.com/stackrox/roxie/internal/logger"
 	"github.com/stackrox/roxie/internal/portforward"
 )
@@ -107,7 +108,6 @@ type Deployer struct {
 	imageCache                *imagecache.ImageCache
 	portForward               *portforward.Manager
 	clusterDefaults           *clusterdefaults.Manager
-	kubectl                   string
 	roxctlVersion             string
 	centralNamespace          string
 	sensorNamespace           string
@@ -135,9 +135,10 @@ type Deployer struct {
 	clusterResourceKinds      map[string]struct{}
 }
 
-type ResourceKindWithName struct {
-	Kind string
-	Name string
+type ResourceToDelete struct {
+	Kind      string
+	Name      string
+	OwnerName string
 }
 
 func (d *Deployer) filterResourceKinds(resourceKinds []string) []string {
@@ -165,12 +166,12 @@ func (d *Deployer) deleteResources(ctx context.Context, namespace string, resour
 		"--grace-period=0",
 	}
 	finalArgs = append(finalArgs, args...)
-	_, err := d.runKubectl(ctx, KubectlOptions{Args: finalArgs})
+	_, err := d.runKubectl(ctx, k8s.KubectlOptions{Args: finalArgs})
 	return err
 }
 
 func (d *Deployer) deleteFinalizers(ctx context.Context, namespace, resourceType, resourceName string) error {
-	_, err := d.runKubectl(ctx, KubectlOptions{
+	_, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args: []string{
 			"-n", namespace, "patch", resourceType, resourceName,
 			"-p", `{"metadata":{"finalizers":null}}`,
@@ -215,13 +216,30 @@ func (d *Deployer) deleteCentralResources(ctx context.Context, wait bool) error
 		return err
 	}
 
-	for _, resource := range []ResourceKindWithName{
-		{Name: "central-db", Kind: "pvc"},
-		{Name: "central-db-backup", Kind: "pvc"},
+	for _, resource := range []ResourceToDelete{
+		{Name: "central-db", Kind: "pvc", OwnerName: centralCrName},
+		{Name: "central-db-backup", Kind: "pvc", OwnerName: centralCrName},
 		{Name: "admin-password", Kind: "secret"},
+		{Name: "scanner-db-password", Kind: "secret", OwnerName: centralCrName},
 	} {
-		err := d.deleteResource(ctx, d.centralNamespace, resource.Kind, resource.Name)
-		if err != nil {
+		d.logger.Dimf("Attempting to delete %s/%s", resource.Kind, resource.Name)
+		if resource.OwnerName != "" {
+			// Avoid deletion if the resource does not have the expected owner.
+			// (e.g. in case central and secured cluster are deployed into the same namespace).
+			obj, err := k8s.RetrieveResourceFromCluster(ctx, d.logger, d.centralNamespace, resource.Kind, resource.Name)
+			if err != nil {
+				if !k8s.IsResourceNotFound(err) {
+					d.logger.Warningf("Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", resource.Kind, resource.Name, err)
+				}
+				continue
+			}
+			if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) {
+				d.logger.Dimf("Skipping deletion of %s/%s: not owned by %s", resource.Kind, resource.Name, resource.OwnerName)
+				continue
+			}
+		}
+
+		if err := d.deleteResource(ctx, d.centralNamespace, resource.Kind, resource.Name); err != nil {
 			return fmt.Errorf("failed to delete %s/%s: %w", resource.Kind, resource.Name, err)
 		}
 	}
@@ -249,7 +267,7 @@ func (d *Deployer) preventCABundleInjection(ctx context.Context) error {
 	}
 
 	d.logger.Info("Removing CNO label from injected-cabundle ConfigMap to prevent CNO from injecting the CA bundle during cleanup")
-	_, err := d.runKubectl(ctx, KubectlOptions{
+	_, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args: []string{
 			"label", "configmap", configMapName, "-n", d.centralNamespace,
 			"config.openshift.io/inject-trusted-cabundle-",
@@ -290,12 +308,29 @@ func (d *Deployer) deleteSecuredClusterResources(ctx context.Context, wait bool)
 		return err
 	}
 
-	for _, resource := range []ResourceKindWithName{
+	for _, resource := range []ResourceToDelete{
 		{Name: "cluster-registration-secret", Kind: "secret"},
-		{Name: "scanner-db-password", Kind: "secret"},
+		// We need to make sure that don't accidentally delete a scanner-db-password belonging to the central CR,
+		// when both are deployed into the same namespace.
+		{Name: "scanner-db-password", Kind: "secret", OwnerName: securedClusterCrName},
 	} {
-		err := d.deleteResource(ctx, d.sensorNamespace, resource.Kind, resource.Name)
-		if err != nil {
+		d.logger.Dimf("Attempting to delete %s/%s", resource.Kind, resource.Name)
+		if resource.OwnerName != "" {
+			// Avoid deletion if the resource does not have the expected owner.
+			// (e.g. in case central and secured cluster are deployed into the same namespace).
+			obj, err := k8s.RetrieveResourceFromCluster(ctx, d.logger, d.sensorNamespace, resource.Kind, resource.Name)
+			if err != nil {
+				if !k8s.IsResourceNotFound(err) {
+					d.logger.Warningf("Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", resource.Kind, resource.Name, err)
+				}
+				continue
+			}
+			if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) {
+				d.logger.Dimf("Skipping deletion of %s/%s: not owned by %s", resource.Kind, resource.Name, resource.OwnerName)
+				continue
+			}
+		}
+		if err := d.deleteResource(ctx, d.sensorNamespace, resource.Kind, resource.Name); err != nil {
 			return fmt.Errorf("failed to delete %s/%s: %w", resource.Kind, resource.Name, err)
 		}
 	}
@@ -477,12 +512,9 @@ func New(log *logger.Logger) (*Deployer, error) {
 		return nil, err
 	}
 
-	kubectl := getKubectl()
-
 	d := &Deployer{
 		logger:                    log,
 		startTime:                 time.Now(),
-		kubectl:                   kubectl,
 		roxctlVersion:             roxctlVersion,
 		centralNamespace:          centralNamespace,
 		sensorNamespace:           sensorNamespace,
@@ -494,7 +526,7 @@ func New(log *logger.Logger) (*Deployer, error) {
 
 	d.dockerAuth = dockerauth.New(log)
 	d.imageCache = imagecache.New(log, "", 20)
-	d.portForward = portforward.New(kubectl, log)
+	d.portForward = portforward.New(k8s.GetKubectl(), log)
 	d.clusterDefaults = clusterdefaults.NewManager(log)
 
 	if password := os.Getenv("ROX_ADMIN_PASSWORD"); password != "" {
@@ -511,11 +543,7 @@ func New(log *logger.Logger) (*Deployer, error) {
 		d.roxCACertFile = caCert
 	}
 
-	ctx, err := getCurrentContext(kubectl)
-	if err != nil {
-		return nil, err
-	}
-	d.kubeContext = ctx
+	d.kubeContext = env.GetCurrentContext()
 
 	clusterResourceKinds, err := d.getClusterResourceKinds()
 	if err != nil {
@@ -530,7 +558,7 @@ func New(log *logger.Logger) (*Deployer, error) {
 }
 
 func (d *Deployer) getClusterResourceKinds() (map[string]struct{}, error) {
-	result, err := d.runKubectl(context.Background(), KubectlOptions{
+	result, err := d.runKubectl(context.Background(), k8s.KubectlOptions{
 		Args: []string{"api-resources", "-o", "name"},
 	})
 	if err != nil {
@@ -754,15 +782,15 @@ func (d *Deployer) ensureNamespaceExists(namespace string) error {
 	}
 
 	d.logger.Infof("Creating namespace %s", namespace)
-	_, err := d.runKubectl(context.Background(), KubectlOptions{
+	_, err := d.runKubectl(context.Background(), k8s.KubectlOptions{
 		Args: []string{"create", "namespace", namespace},
 	})
 	if err != nil {
 		return fmt.Errorf("failed to create namespace: %w", err)
 	}
 
 	// Label namespace as managed by roxie since we just created it
-	_, err = d.runKubectl(context.Background(), KubectlOptions{
+	_, err = d.runKubectl(context.Background(), k8s.KubectlOptions{
 		Args: []string{"label", "namespace", namespace,
 			"app.kubernetes.io/managed-by=roxie", "--overwrite"},
 	})
@@ -774,7 +802,7 @@ func (d *Deployer) ensureNamespaceExists(namespace string) error {
 }
 
 func (d *Deployer) namespaceExists(namespace string) bool {
-	_, err := d.runKubectl(context.Background(), KubectlOptions{
+	_, err := d.runKubectl(context.Background(), k8s.KubectlOptions{
 		Args: []string{"get", "namespace", namespace},
 	})
 	return err == nil
@@ -824,13 +852,6 @@ func checkRequiredTools() error {
 	return nil
 }
 
-func getKubectl() string {
-	if kubectl := os.Getenv("ORCH_CMD"); kubectl != "" {
-		return kubectl
-	}
-	return "kubectl"
-}
-
 func getRoxctlVersion() (string, error) {
 	cmd := exec.Command("roxctl", "version")
 	output, err := cmd.Output()
@@ -852,16 +873,6 @@ func getRoxctlVersion() (string, error) {
 	return version, nil
 }
 
-func getCurrentContext(kubectl string) (string, error) {
-	cmd := exec.Command(kubectl, "config", "current-context")
-	output, err := cmd.Output()
-	if err != nil {
-		return "", fmt.Errorf("failed to get current context: %w", err)
-	}
-
-	return strings.TrimSpace(string(output)), nil
-}
-
 func generatePassword() string {
 	const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"
 	const passwordLength = 20
@@ -948,7 +959,7 @@ func (d *Deployer) maybeAddPauseReconcileAnnotation(ctx context.Context, resourc
 }
 
 func (d *Deployer) doesResourceExist(ctx context.Context, resourceType, resourceName, namespace string) bool {
-	_, err := d.runKubectl(ctx, KubectlOptions{
+	_, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args: []string{
 			"get", resourceType, resourceName,
 			"-n", namespace,
@@ -958,7 +969,7 @@ func (d *Deployer) doesResourceExist(ctx context.Context, resourceType, resource
 }
 
 func (d *Deployer) addPauseReconcileAnnotation(ctx context.Context, resourceType, resourceName, namespace string) error {
-	_, err := d.runKubectl(ctx, KubectlOptions{
+	_, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args: []string{
 			"annotate", resourceType, resourceName,
 			"-n", namespace,
@@ -1155,7 +1166,7 @@ func (d *Deployer) PrintCentralDeploymentSummary() {
 
 // checkDeploymentProgressInNamespace checks for deployment state changes in a specific namespace and reports them
 func (d *Deployer) checkDeploymentProgressInNamespace(ctx context.Context, namespace string, seenDeployments map[string]string) {
-	result, err := d.runKubectl(ctx, KubectlOptions{
+	result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args: []string{"get", "deployments", "-n", namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{'|'}{.status.replicas}{'|'}{.status.readyReplicas}{'|'}{.status.availableReplicas}{'\\n'}{end}"},
 	})
 	if err != nil {
@@ -1200,7 +1211,7 @@ func (d *Deployer) checkDeploymentProgressInNamespace(ctx context.Context, names
 
 // checkPodProgressInNamespace checks for pod state changes in a specific namespace and reports them
 func (d *Deployer) checkPodProgressInNamespace(ctx context.Context, namespace string, seenPods map[string]string) {
-	result, err := d.runKubectl(ctx, KubectlOptions{
+	result, err := d.runKubectl(ctx, k8s.KubectlOptions{
 		Args: []string{"get", "pods", "-n", namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{'|'}{.status.phase}{'|'}{.status.containerStatuses[0].ready}{'\\n'}{end}"},
 	})
 	if err != nil {