Dump cluster state on failure

Author: Moritz Clasmeier

tests/e2e/basic_test.go

@@ -11,6 +11,8 @@ import (
 
 // TestDeployBothSimple tests deploying both components together (simplest scenario)
 func TestDeployBothSimple(t *testing.T) {
+	dumpClusterOnFailure(t)
+
 	// Create temporary envrc file
 	envrcFile, err := os.CreateTemp(t.TempDir(), ".envrc.roxie-test-*")
 	if err != nil {

tests/e2e/e2e_test.go

@@ -39,6 +39,8 @@ func TestMain(m *testing.M) {
 }
 
 func TestDeployBothComponentsTogetherInSingleNamespace(t *testing.T) {
+	dumpClusterOnFailure(t)
+
 	// Create temporary envrc file.
 	envrcFile, err := os.CreateTemp(t.TempDir(), ".envrc.roxie-test-*")
 	if err != nil {

tests/e2e/helpers.go

@@ -239,6 +239,115 @@ func verifySecuredClusterNotInstalled(t *testing.T, namespace string) {
 	}
 }
 
+var clusterDumpNamespaces = []string{
+	"rhacs-operator-system",
+	"acs-central",
+	"acs-sensor",
+	"stackrox",
+}
+
+func dumpClusterOnFailure(t *testing.T) {
+	t.Helper()
+	t.Cleanup(func() {
+		if !t.Failed() {
+			return
+		}
+		dumpClusterResources(t)
+	})
+}
+
+func dumpClusterResources(t *testing.T) {
+	t.Helper()
+	fmt.Fprintf(os.Stderr, "=== CLUSTER RESOURCE DUMP (test %s failed) ===\n", t.Name())
+
+	runKubectlDump("get", "namespaces")
+
+	for _, ns := range clusterDumpNamespaces {
+		fmt.Fprintf(os.Stderr, "--- Namespace: %s ---\n", ns)
+		runKubectlDump("get", "pods", "-n", ns, "-o", "wide")
+		runKubectlDump("describe", "pods", "-n", ns)
+		runKubectlDump("get", "deployments", "-n", ns, "-o", "wide")
+		runKubectlDump("describe", "deployments", "-n", ns)
+		runKubectlDump("get", "daemonsets", "-n", ns, "-o", "wide")
+		runKubectlDump("describe", "daemonsets", "-n", ns)
+		runKubectlDump("get", "events", "-n", ns, "--sort-by=.lastTimestamp")
+		dumpLogsForFailingPods(ns)
+	}
+
+	dumpACSCustomResources()
+	dumpOLMResources()
+
+	fmt.Fprintln(os.Stderr, "=== END CLUSTER RESOURCE DUMP ===")
+}
+
+func dumpACSCustomResources() {
+	fmt.Fprintln(os.Stderr, "--- ACS Custom Resources ---")
+	for _, ns := range clusterDumpNamespaces {
+		runKubectlDump("get", "centrals.platform.stackrox.io", "-n", ns, "-o", "yaml")
+		runKubectlDump("get", "securedclusters.platform.stackrox.io", "-n", ns, "-o", "yaml")
+	}
+}
+
+func dumpOLMResources() {
+	cmd := exec.Command("kubectl", "api-resources", "--api-group=operators.coreos.com", "-o", "name")
+	output, err := cmd.Output()
+	if err != nil || strings.TrimSpace(string(output)) == "" {
+		fmt.Fprintln(os.Stderr, "[dump] OLM not installed, skipping OLM resource dump")
+		return
+	}
+
+	fmt.Fprintln(os.Stderr, "--- OLM Resources ---")
+	olmNamespace := "rhacs-operator-system"
+	runKubectlDump("get", "subscriptions.operators.coreos.com", "-n", olmNamespace, "-o", "wide")
+	runKubectlDump("describe", "subscriptions.operators.coreos.com", "-n", olmNamespace)
+	runKubectlDump("get", "installplans.operators.coreos.com", "-n", olmNamespace, "-o", "wide")
+	runKubectlDump("describe", "installplans.operators.coreos.com", "-n", olmNamespace)
+	runKubectlDump("get", "catalogsources.operators.coreos.com", "-n", olmNamespace, "-o", "wide")
+	runKubectlDump("describe", "catalogsources.operators.coreos.com", "-n", olmNamespace)
+	runKubectlDump("get", "clusterserviceversions.operators.coreos.com", "-n", olmNamespace, "-o", "wide")
+	runKubectlDump("describe", "clusterserviceversions.operators.coreos.com", "-n", olmNamespace)
+	runKubectlDump("get", "operatorgroups.operators.coreos.com", "-n", olmNamespace, "-o", "wide")
+	runKubectlDump("describe", "operatorgroups.operators.coreos.com", "-n", olmNamespace)
+}
+
+func runKubectlDump(args ...string) {
+	fmt.Fprintf(os.Stderr, "## kubectl %s\n", strings.Join(args, " "))
+	cmd := exec.Command("kubectl", args...)
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "kubectl failed: %v\n", err)
+	}
+	fmt.Fprintln(os.Stderr)
+}
+
+func dumpLogsForFailingPods(namespace string) {
+	cmd := exec.Command("kubectl", "get", "pods", "-n", namespace,
+		"-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\t\"}{.status.phase}{\"\\n\"}{end}")
+	output, err := cmd.Output()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "[dump] failed to list pods in %s: %v\n", namespace, err)
+		return
+	}
+
+	for line := range strings.SplitSeq(strings.TrimSpace(string(output)), "\n") {
+		if line == "" {
+			continue
+		}
+		parts := strings.SplitN(line, "\t", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		podName, phase := parts[0], parts[1]
+		if phase == "Running" || phase == "Succeeded" {
+			continue
+		}
+		fmt.Fprintf(os.Stderr, "[dump] logs for pod %s/%s (phase=%s):\n", namespace, podName, phase)
+		runKubectlDump("logs", "-n", namespace, podName, "--all-containers", "--tail=100")
+		runKubectlDump("logs", "-n", namespace, podName, "--all-containers", "--previous", "--tail=50")
+	}
+}
+
 func verifyAnnotation(t *testing.T, resourceType, resourceName, namespace, annotationKey, expectedValue string) {
 	t.Helper()
 

tests/e2e/olm_switch_test.go

@@ -65,6 +65,8 @@ func verifyOperatorDeploymentExists(t *testing.T) {
 
 // TestOLMToNonOLMSwitch tests switching from OLM operator to non-OLM operator
 func TestOLMToNonOLMSwitch(t *testing.T) {
+	dumpClusterOnFailure(t)
+
 	if os.Getenv("SKIP_OLM_TESTS") != "" {
 		t.Skip("SKIP_OLM_TESTS is set")
 	}
@@ -113,6 +115,8 @@ func TestOLMToNonOLMSwitch(t *testing.T) {
 
 // TestNonOLMToOLMSwitch tests switching from non-OLM operator to OLM operator
 func TestNonOLMToOLMSwitch(t *testing.T) {
+	dumpClusterOnFailure(t)
+
 	if os.Getenv("SKIP_OLM_TESTS") != "" {
 		t.Skip("SKIP_OLM_TESTS is set")
 	}
@@ -161,6 +165,8 @@ func TestNonOLMToOLMSwitch(t *testing.T) {
 
 // TestOLMOperatorVersionUpgrade tests that OLM operator version mismatches trigger teardown and redeploy
 func TestOLMOperatorVersionUpgrade(t *testing.T) {
+	dumpClusterOnFailure(t)
+
 	if os.Getenv("SKIP_OLM_TESTS") != "" {
 		t.Skip("SKIP_OLM_TESTS is set")
 	}
@@ -223,6 +229,8 @@ func TestOLMOperatorVersionUpgrade(t *testing.T) {
 
 // TestSecuredClusterWithOLMSwitch tests that secured-cluster deployment also respects OLM mode switches
 func TestSecuredClusterWithOLMSwitch(t *testing.T) {
+	dumpClusterOnFailure(t)
+
 	if os.Getenv("SKIP_OLM_TESTS") != "" {
 		t.Skip("SKIP_OLM_TESTS is set")
 	}