Merge pull request #318 from NearNodeFlash:wait-for-prerequisites-kind

Improve KIND deployment reliability

Author: ajfloeder

cmd/main.go

@@ -104,6 +104,11 @@ func (cmd *DeployCmd) Run(ctx *Context) error {
 			return nil
 		}
 
+		// Wait for prerequisites that this module needs.
+		if err := waitForModulePrereqs(ctx, module); err != nil {
+			return err
+		}
+
 		if err := createSystemConfigFromSOS(ctx, system, module); err != nil {
 			return err
 		}
@@ -1044,6 +1049,100 @@ func getExampleOverlay(ctx *Context, system *config.System, module string) (stri
 	return "", nil
 }
 
+// Modules that require cert-manager webhook to be ready before deploying.
+var modulesCertManager = []string{
+	"lustre-fs-operator",
+	"dws",
+	"nnf-sos",
+}
+
+// Modules that require the DWS controller-manager to be ready before deploying,
+// because they use DWS CRDs that have a conversion webhook served by DWS.
+var modulesDWSController = []string{
+	"nnf-sos",
+	"nnf-dm",
+}
+
+// waitForModulePrereqs waits for prerequisite services that a module depends on.
+func waitForModulePrereqs(ctx *Context, module string) error {
+	if ctx.DryRun {
+		return nil
+	}
+
+	for _, m := range modulesCertManager {
+		if m == module {
+			if err := waitForCertManagerWebhook(ctx); err != nil {
+				return err
+			}
+			break
+		}
+	}
+
+	for _, m := range modulesDWSController {
+		if m == module {
+			if err := waitForDWSControllerManager(ctx); err != nil {
+				return err
+			}
+			break
+		}
+	}
+
+	return nil
+}
+
+// waitForCertManagerWebhook waits for the cert-manager webhook to be ready and
+// functional. Modules that create cert-manager Certificate or Issuer resources
+// need this webhook to be operational.
+func waitForCertManagerWebhook(ctx *Context) error {
+	fmt.Println("  Waiting for cert-manager webhook...")
+	cmd := exec.Command("kubectl", "wait", "deploy", "-n", "cert-manager",
+		"--timeout=180s", "cert-manager-webhook",
+		"--for", "jsonpath={.status.availableReplicas}=1")
+	if _, err := runCommand(ctx, cmd); err != nil {
+		return fmt.Errorf("cert-manager webhook deployment not ready: %w", err)
+	}
+
+	// The deployment being available doesn't guarantee the webhook is
+	// functional. Verify by attempting a server-side dry-run of an Issuer.
+	fmt.Println("  Verifying cert-manager webhook is functional...")
+	for attempts := 0; attempts < 60; attempts++ {
+		cmd = exec.Command("kubectl", "apply", "--dry-run=server", "-f", "-")
+		cmd.Stdin = strings.NewReader(`apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: deploy-dryrun-test
+  namespace: default
+spec:
+  selfSigned: {}
+`)
+		if out, err := cmd.CombinedOutput(); err == nil {
+			_ = out
+			break
+		}
+		if attempts == 59 {
+			return fmt.Errorf("cert-manager webhook not functional after 120s")
+		}
+		time.Sleep(2 * time.Second)
+	}
+
+	return nil
+}
+
+// waitForDWSControllerManager waits for the DWS controller-manager to be fully
+// ready. The DWS controller-manager serves the CRD conversion webhook needed by
+// modules that access DWS custom resources with multiple API versions.
+func waitForDWSControllerManager(ctx *Context) error {
+	fmt.Println("  Waiting for DWS controller-manager...")
+	cmd := exec.Command("kubectl", "wait", "deploy", "-n", "dws-system",
+		"--timeout=180s", "dws-controller-manager",
+		"--for", "jsonpath={.status.availableReplicas}=1")
+	if _, err := runCommand(ctx, cmd); err != nil {
+		return fmt.Errorf("DWS controller-manager not ready: %w", err)
+	}
+
+	return nil
+}
+
 func deployModule(ctx *Context, system *config.System, module string) error {
 
 	cmd := exec.Command("make", "deploy")

dws

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2021-2024 Hewlett Packard Enterprise Development LP
+# Copyright 2021-2026 Hewlett Packard Enterprise Development LP
 # Other additional copyright holders may be indicated within.
 #
 # The entirety of this work is licensed under the Apache License,
@@ -32,6 +32,51 @@ function clear_mock_fs {
     fi
 }
 
+function inject_ca_certs {
+    local certs="$KIND_CA_CERTS"
+    local tmp_certs=""
+
+    # If KIND_CA_CERTS is not set, auto-extract system CA certs.
+    if [[ -z "$certs" ]]; then
+        tmp_certs=$(mktemp /tmp/kind-ca-certs.XXXXXX.pem)
+        if [[ "$(uname)" == "Darwin" ]]; then
+            if [[ -f /Library/Keychains/System.keychain ]]; then
+                security find-certificate -a -p /Library/Keychains/System.keychain > "$tmp_certs" 2>/dev/null
+            fi
+        else
+            # Linux: copy the system CA bundle.
+            if [[ -f /etc/ssl/certs/ca-certificates.crt ]]; then
+                cp /etc/ssl/certs/ca-certificates.crt "$tmp_certs"
+            elif [[ -f /etc/pki/tls/certs/ca-bundle.crt ]]; then
+                cp /etc/pki/tls/certs/ca-bundle.crt "$tmp_certs"
+            fi
+        fi
+
+        if [[ ! -s "$tmp_certs" ]]; then
+            echo "WARNING: Could not extract system CA certificates. Set KIND_CA_CERTS to a PEM file if image pulls fail with TLS errors."
+            rm -f "$tmp_certs"
+            return
+        fi
+        certs="$tmp_certs"
+    fi
+
+    if [[ ! -f "$certs" ]]; then
+        echo "ERROR: KIND_CA_CERTS file not found: $certs"
+        return 1
+    fi
+
+    echo "Injecting CA certificates from $certs into KIND nodes..."
+    for node in $(kind get nodes); do
+        docker cp "$certs" "$node:/usr/local/share/ca-certificates/extra-ca-certs.crt"
+        docker exec "$node" update-ca-certificates
+        docker exec "$node" systemctl restart containerd
+        echo "  $node: done"
+    done
+
+    # Clean up temp file if we created one.
+    [[ -n "$tmp_certs" ]] && rm -f "$tmp_certs"
+}
+
 function create_cluster {
     CONFIG=kind-config.yaml
 
@@ -92,6 +137,12 @@ EOF
 
     kind create cluster --wait 60s --image=kindest/node:v1.31.2 --config $CONFIG
 
+    # If corporate/custom CA certificates are available, inject them into
+    # each KIND node so containerd can pull from registries behind a TLS
+    # intercepting proxy. Set KIND_CA_CERTS to a PEM file path, or it
+    # defaults to /tmp/corporate-certs.pem.
+    inject_ca_certs
+
     # Use the same init routines that we use on real hardware.
     # This applies taints and labels to rabbit nodes, and installs other
     # services that rabbit software requires.