Update

Author: jgwest

test/helper/application_status.go

@@ -173,7 +173,8 @@ func DeleteNamespace(k8sClient client.Client, nsToDelete string) error {
 			GinkgoT().Errorf("Unable to list ArgoCDs %v", err)
 			// Report failure, but still continue
 		}
-		for _, item := range list.Items {
+		for idx := range list.Items {
+			item := list.Items[idx]
 
 			if len(item.Finalizers) == 0 {
 				continue

test/openshift/e2e/ginkgo/fixture/fixture.go

@@ -28,6 +28,7 @@ import (
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
 	deploymentFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
+	osFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/os"
 	subscriptionFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/subscription"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
 	appsv1 "k8s.io/api/apps/v1"
@@ -785,3 +786,61 @@ func updateWithoutConflict(obj client.Object, modify func(client.Object)) error
 
 	return err
 }
+
+// DebugOutputOperatorLogs can be used to debug a failing test: it will output the operator logs
+func DebugOutputOperatorLogs() {
+
+	k8sClient, _, err := utils.GetE2ETestKubeClientWithError()
+	if err != nil {
+		GinkgoWriter.Println(err)
+		return
+	}
+
+	// List all pods on the cluster
+	var podList corev1.PodList
+	if err := k8sClient.List(context.Background(), &podList); err != nil {
+		GinkgoWriter.Println(err)
+		return
+	}
+
+	// Look specifically for operator pod
+	matchingPods := []corev1.Pod{}
+	for idx := range podList.Items {
+		pod := podList.Items[idx]
+		if strings.Contains(pod.Name, "openshift-gitops-operator-controller-manager") {
+			matchingPods = append(matchingPods, pod)
+		}
+	}
+
+	if len(matchingPods) == 0 {
+		// This can happen when the operator is not running on the cluster
+		GinkgoWriter.Println("DebugOutputOperatorLogs was called, but no pods were found.")
+		return
+	}
+
+	if len(matchingPods) != 1 {
+		GinkgoWriter.Println("unexpected number of operator pods", matchingPods)
+		return
+	}
+
+	// Extract operator logs
+	kubectlLogOutput, err := osFixture.ExecCommandWithOutputParam(false, "kubectl", "logs", "pod/"+matchingPods[0].Name, "manager", "-n", matchingPods[0].Namespace)
+	if err != nil {
+		GinkgoWriter.Println("unable to extract operator logs", err)
+		return
+	}
+
+	// Output only the last 1000 lines
+	lines := strings.Split(kubectlLogOutput, "\n")
+
+	count := 0
+	output := []string{}
+	for x := len(lines) - 1; x >= 0 && count < 1000; x-- {
+		output = append(lines[x:x+1], output...)
+		count++
+	}
+
+	GinkgoWriter.Println("Log output from operator pod:")
+	GinkgoWriter.Println(output)
+
+}

test/openshift/e2e/ginkgo/fixture/os/fixture.go

@@ -16,10 +16,16 @@ func ExecCommandWithOutputParam(printOutput bool, cmdArgs ...string) (string, er
 
 	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
 
-	output, err := cmd.CombinedOutput()
+	outputBytes, err := cmd.CombinedOutput()
+
+	var output string
+	if outputBytes != nil {
+		output = string(outputBytes)
+	}
+
 	if printOutput {
-		GinkgoWriter.Println(string(output))
+		GinkgoWriter.Println(output)
 	}
 
-	return string(output), err
+	return output, err
 }

test/openshift/e2e/ginkgo/parallel/1-004_beta_to_alpha_conversion_test.go

@@ -119,13 +119,13 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			Expect(k8sClient.Delete(ctx, argoCDAlpha1)).To(Succeed())
 
 			By("verifying ArgoCD CR no long exists via v1alpha1 API")
-			Expect(argoCDAlpha1).To(k8sFixture.NotExistByName())
+			Eventually(argoCDAlpha1).Should(k8sFixture.NotExistByName())
 
 			By("verifying ArgoCD CR no long exists via v1beta1 API")
 			Expect(argoCDBeta1).To(k8sFixture.NotExistByName())
 
 			By("verifying dex deployment is deleted as well")
-			Eventually(dexDeployment).ShouldNot(k8sFixture.ExistByName())
+			Eventually(dexDeployment).Should(k8sFixture.NotExistByName())
 
 		})
 

test/openshift/e2e/ginkgo/parallel/1-073_validate_rhsso_test.go

@@ -221,7 +221,16 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			Expect(findLineInOutput("rootca", oidcConfigContents)).Should(Equal("'---BEGIN---END---'"))
 
 			By("extracting the access token for keycloak, to allow us to issue API commands")
-			output, err := osFixture.ExecCommandWithOutputParam(false, "curl", "-d", "client_id=admin-cli", "-d", "username="+string(keycloakSecret.Data["SSO_USERNAME"]), "-d", "password="+string(keycloakSecret.Data["SSO_PASSWORD"]), "-d", "grant_type=password", "https://"+keycloakRouteHost+"/auth/realms/master/protocol/openid-connect/token", "-k")
+
+			ssoUsername, exists := keycloakSecret.Data["SSO_USERNAME"]
+			Expect(exists).To(BeTrue())
+			Expect(ssoUsername).ToNot(BeNil())
+
+			ssoPassword, exists := keycloakSecret.Data["SSO_PASSWORD"]
+			Expect(exists).To(BeTrue())
+			Expect(ssoPassword).ToNot(BeNil())
+
+			output, err := osFixture.ExecCommandWithOutputParam(false, "curl", "-d", "client_id=admin-cli", "-d", "username="+string(ssoUsername), "-d", "password="+string(ssoPassword), "-d", "grant_type=password", "https://"+keycloakRouteHost+"/auth/realms/master/protocol/openid-connect/token", "-k")
 			Expect(err).ToNot(HaveOccurred())
 
 			// Extract the JSON part of the output
@@ -230,7 +239,8 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			var jsonData map[string]any
 			Expect(json.Unmarshal([]byte(output), &jsonData)).ToNot(HaveOccurred())
 
-			accessToken := jsonData["access_token"].(string)
+			accessToken, success := jsonData["access_token"].(string)
+			Expect(success).To(BeTrue())
 
 			By("executing the CURL command to verify the realm and client creation")
 

test/openshift/e2e/ginkgo/parallel/1-083_validate_resource_customization_subkeys_test.go

@@ -165,8 +165,8 @@ spec:
   help.chatUrl: ""
   kustomize.buildOptions: ""
   oidc.config: ""
-  repositories: ""
-  repository.credentials: ""
+# repositories: ""
+# repository.credentials: ""
   resource.customizations.actions.apps_Deployment: |
     discovery.lua: |
     actions = {}

test/openshift/e2e/ginkgo/parallel/1-090_validate_permissions_test.go

@@ -49,6 +49,11 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 
 		It("ensure the GitOps Operator CSV has the correct cluster permissions and gitopsservices CRD has expected properties", func() {
 
+			if fixture.EnvCI() {
+				Skip("AFAICT CSV does not exist when running E2E tests from gitops-operator repo")
+				return
+			}
+
 			if fixture.EnvLocalRun() || fixture.EnvNonOLM() {
 				Skip("CSV does not exist in the local run or non-OLM case, so we skip")
 				return
@@ -711,7 +716,7 @@ spec:
 				}
 			}
 			By("if more than one possible CSV is found, we will fail.")
-			Expect(gitopsCSVsFound).To(HaveLen(1), fmt.Sprintf("multiple CSVs were found: %v", gitopsCSVsFound))
+			Expect(gitopsCSVsFound).To(HaveLen(1), fmt.Sprintf("Exactly one CSV should found: %v", gitopsCSVsFound))
 
 			actualCsv := &olmv1alpha1.ClusterServiceVersion{
 				ObjectMeta: metav1.ObjectMeta{

test/openshift/e2e/ginkgo/sequential/1-036_validate_role_rolebinding_for_source_namespace_test.go

@@ -33,6 +33,12 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			ctx = context.Background()
 		})
 
+		AfterEach(func() {
+			if CurrentSpecReport().Failed() {
+				fixture.DebugOutputOperatorLogs()
+			}
+		})
+
 		It("verifies that ArgoCD CR '.spec.sourceNamespaces' field wildcard-matching matches and manages only namespaces which match the wildcard", func() {
 
 			By("creating test NS")

…te_handle_terminating_namespaces_test.go …te_handle_terminating_namespaces_test.gotest/openshift/e2e/ginkgo/parallel/1-102_validate_handle_terminating_namespaces_test.go renamed to test/openshift/e2e/ginkgo/sequential/1-102_validate_handle_terminating_namespaces_test.go test/openshift/e2e/ginkgo/parallel/1-102_validate_handle_terminating_namespaces_test.go renamed to test/openshift/e2e/ginkgo/sequential/1-102_validate_handle_terminating_namespaces_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package parallel
+package sequential
 
 import (
 	"context"
@@ -46,7 +46,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 		)
 
 		BeforeEach(func() {
-			fixture.EnsureParallelCleanSlate()
+			fixture.EnsureSequentialCleanSlate()
 
 			k8sClient, _ = fixtureUtils.GetE2ETestKubeClient()
 			ctx = context.Background()