ci(e2e): harden release gate setup (#462)

Signed-off-by: Roel de Cort <roel.decort@adfinis.com>

Author: dc-tec

.github/actions/prepare-e2e-lane/action.yml

@@ -95,17 +95,39 @@ runs:
       run: |
         set -euo pipefail
 
-        docker pull "${MANAGER_IMAGE}"
-        docker pull "${CONFIG_INIT_IMAGE}"
+        pull_image() {
+          local image="$1"
+          local max_attempts=5
+          local delay_seconds=5
+          local attempt
+
+          for (( attempt = 1; attempt <= max_attempts; attempt++ )); do
+            if docker pull "${image}"; then
+              return 0
+            fi
+
+            if (( attempt == max_attempts )); then
+              echo "docker pull failed for ${image} after ${max_attempts} attempts" >&2
+              return 1
+            fi
+
+            echo "docker pull failed for ${image} (attempt ${attempt}/${max_attempts}); retrying in ${delay_seconds}s" >&2
+            sleep "${delay_seconds}"
+            delay_seconds=$(( delay_seconds * 2 ))
+          done
+        }
+
+        pull_image "${MANAGER_IMAGE}"
+        pull_image "${CONFIG_INIT_IMAGE}"
 
         if [[ "${LOAD_BACKUP_EXECUTOR_IMAGE}" == "true" ]]; then
-          docker pull "${BACKUP_EXECUTOR_IMAGE}"
+          pull_image "${BACKUP_EXECUTOR_IMAGE}"
         fi
 
         if [[ "${LOAD_UPGRADE_EXECUTOR_IMAGE}" == "true" ]] && [[ -n "${HARDENED_UPGRADE_EXECUTOR_IMAGE}" ]]; then
-          docker pull "${HARDENED_UPGRADE_EXECUTOR_IMAGE}"
+          pull_image "${HARDENED_UPGRADE_EXECUTOR_IMAGE}"
         elif [[ "${LOAD_UPGRADE_EXECUTOR_IMAGE}" == "true" ]]; then
-          docker pull "${UPGRADE_EXECUTOR_IMAGE}"
+          pull_image "${UPGRADE_EXECUTOR_IMAGE}"
         fi
 
     - name: Create Kind cluster
@@ -135,6 +157,28 @@ runs:
       run: |
         set -euo pipefail
 
+        pull_image() {
+          local image="$1"
+          local max_attempts=5
+          local delay_seconds=5
+          local attempt
+
+          for (( attempt = 1; attempt <= max_attempts; attempt++ )); do
+            if docker pull "${image}"; then
+              return 0
+            fi
+
+            if (( attempt == max_attempts )); then
+              echo "docker pull failed for ${image} after ${max_attempts} attempts" >&2
+              return 1
+            fi
+
+            echo "docker pull failed for ${image} (attempt ${attempt}/${max_attempts}); retrying in ${delay_seconds}s" >&2
+            sleep "${delay_seconds}"
+            delay_seconds=$(( delay_seconds * 2 ))
+          done
+        }
+
         resolve_image() {
           local output_name="$1"
           local preload="$2"
@@ -147,7 +191,7 @@ runs:
           fi
 
           if [[ "${PRELOAD_STORAGE_EMULATORS}" == "true" ]] && [[ "${preload}" == "true" ]]; then
-            docker pull "${source_image}"
+            pull_image "${source_image}"
             docker tag "${source_image}" "${loaded_image}"
             kind load docker-image "${loaded_image}" --name "${KIND_CLUSTER}"
             echo "${output_name}=${loaded_image}" >> "${GITHUB_OUTPUT}"

test/e2e/Cluster_Profile_Hardened_test.go

@@ -76,6 +76,42 @@ var _ = Describe("Hardened profile (External TLS + Transit auto-unseal + SelfIni
 		}, framework.DefaultWaitTimeout, framework.DefaultPollInterval).Should(Succeed())
 	}
 
+	waitForNetworkPolicy := func(name types.NamespacedName, timeout, pollInterval time.Duration) error {
+		deadline := time.Now().Add(timeout)
+		var lastErr error
+
+		for {
+			np := &networkingv1.NetworkPolicy{}
+			err := c.Get(ctx, name, np)
+			if err == nil {
+				return nil
+			}
+			if !apierrors.IsNotFound(err) {
+				return err
+			}
+			lastErr = err
+
+			if time.Now().After(deadline) {
+				return fmt.Errorf("timed out waiting for NetworkPolicy %s/%s: %w", name.Namespace, name.Name, lastErr)
+			}
+
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(pollInterval):
+			}
+		}
+	}
+
+	dumpNetworkPolicyDiagnostics := func(namespace, clusterName string) {
+		_, _ = fmt.Fprintf(GinkgoWriter, "\n========== NetworkPolicy Diagnostics (%s/%s) ==========\n", namespace, clusterName)
+		dumpKubectlOutput("get", "openbaocluster", clusterName, "-n", namespace, "-o", "yaml")
+		dumpKubectlOutput("get", "networkpolicies", "-n", namespace, "-o", "wide")
+		dumpKubectlOutput("get", "pods", "-n", namespace, "-l", fmt.Sprintf("%s=%s", constants.LabelOpenBaoCluster, clusterName), "-o", "wide")
+		dumpKubectlOutput("get", "events", "-n", namespace, "--sort-by=.lastTimestamp")
+		dumpKubectlOutput("logs", "deployment/openbao-operator-controller", "-n", operatorNamespace, "--tail=400")
+	}
+
 	ensureTransitTokenSecret := func() {
 		By("creating transit token secret with CA certificate for TLS verification")
 		infraBaoCACert, err := e2ehelpers.ReadInfraBaoTLSCACert(ctx, c, f.Namespace, infraBaoName)
@@ -409,11 +445,12 @@ var _ = Describe("Hardened profile (External TLS + Transit auto-unseal + SelfIni
 		_, _ = fmt.Fprintf(GinkgoWriter, "OpenBaoCluster %q observed by API server\n", clusterName)
 
 		By("verifying NetworkPolicy was created")
-		Eventually(func(g Gomega) {
-			np := &networkingv1.NetworkPolicy{}
-			npName := types.NamespacedName{Name: clusterName + "-network-policy", Namespace: f.Namespace}
-			g.Expect(c.Get(ctx, npName, np)).To(Succeed())
-		}, 30*time.Second, 2*time.Second).Should(Succeed())
+		npName := types.NamespacedName{Name: clusterName + "-network-policy", Namespace: f.Namespace}
+		err = waitForNetworkPolicy(npName, framework.DefaultWaitTimeout, framework.DefaultPollInterval)
+		if err != nil {
+			dumpNetworkPolicyDiagnostics(f.Namespace, clusterName)
+		}
+		Expect(err).NotTo(HaveOccurred())
 		_, _ = fmt.Fprintf(GinkgoWriter, "NetworkPolicy created successfully\n")
 
 		By("checking for prerequisite resources (ConfigMap and TLS Secrets)")