diff --git a/cmd/main.go b/cmd/main.go index 89e322f..ede6bee 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -104,6 +104,11 @@ func (cmd *DeployCmd) Run(ctx *Context) error { return nil } + // Wait for prerequisites that this module needs. + if err := waitForModulePrereqs(ctx, module); err != nil { + return err + } + if err := createSystemConfigFromSOS(ctx, system, module); err != nil { return err } @@ -1044,6 +1049,100 @@ func getExampleOverlay(ctx *Context, system *config.System, module string) (stri return "", nil } +// Modules that require cert-manager webhook to be ready before deploying. +var modulesCertManager = []string{ + "lustre-fs-operator", + "dws", + "nnf-sos", +} + +// Modules that require the DWS controller-manager to be ready before deploying, +// because they use DWS CRDs that have a conversion webhook served by DWS. +var modulesDWSController = []string{ + "nnf-sos", + "nnf-dm", +} + +// waitForModulePrereqs waits for prerequisite services that a module depends on. +func waitForModulePrereqs(ctx *Context, module string) error { + if ctx.DryRun { + return nil + } + + for _, m := range modulesCertManager { + if m == module { + if err := waitForCertManagerWebhook(ctx); err != nil { + return err + } + break + } + } + + for _, m := range modulesDWSController { + if m == module { + if err := waitForDWSControllerManager(ctx); err != nil { + return err + } + break + } + } + + return nil +} + +// waitForCertManagerWebhook waits for the cert-manager webhook to be ready and +// functional. Modules that create cert-manager Certificate or Issuer resources +// need this webhook to be operational. +func waitForCertManagerWebhook(ctx *Context) error { + fmt.Println(" Waiting for cert-manager webhook...") + cmd := exec.Command("kubectl", "wait", "deploy", "-n", "cert-manager", + "--timeout=180s", "cert-manager-webhook", + "--for", "jsonpath={.status.availableReplicas}=1") + if _, err := runCommand(ctx, cmd); err != nil { + return fmt.Errorf("cert-manager webhook deployment not ready: %w", err) + } + + // The deployment being available doesn't guarantee the webhook is + // functional. Verify by attempting a server-side dry-run of an Issuer. + fmt.Println(" Verifying cert-manager webhook is functional...") + for attempts := 0; attempts < 60; attempts++ { + cmd = exec.Command("kubectl", "apply", "--dry-run=server", "-f", "-") + cmd.Stdin = strings.NewReader(`apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: deploy-dryrun-test + namespace: default +spec: + selfSigned: {} +`) + if out, err := cmd.CombinedOutput(); err == nil { + _ = out + break + } + if attempts == 59 { + return fmt.Errorf("cert-manager webhook not functional after 120s") + } + time.Sleep(2 * time.Second) + } + + return nil +} + +// waitForDWSControllerManager waits for the DWS controller-manager to be fully +// ready. The DWS controller-manager serves the CRD conversion webhook needed by +// modules that access DWS custom resources with multiple API versions. +func waitForDWSControllerManager(ctx *Context) error { + fmt.Println(" Waiting for DWS controller-manager...") + cmd := exec.Command("kubectl", "wait", "deploy", "-n", "dws-system", + "--timeout=180s", "dws-controller-manager", + "--for", "jsonpath={.status.availableReplicas}=1") + if _, err := runCommand(ctx, cmd); err != nil { + return fmt.Errorf("DWS controller-manager not ready: %w", err) + } + + return nil +} + func deployModule(ctx *Context, system *config.System, module string) error { cmd := exec.Command("make", "deploy") diff --git a/dws b/dws index cf9255c..4e24ce7 160000 --- a/dws +++ b/dws @@ -1 +1 @@ -Subproject commit cf9255cfc6a772a6400dbc3ee172816d0e598a88 +Subproject commit 4e24ce7ae3727ff0c5cfca274c8c847cc84fdf05 diff --git a/lustre-fs-operator b/lustre-fs-operator index 4b4ec57..7ebec7d 160000 --- a/lustre-fs-operator +++ b/lustre-fs-operator @@ -1 +1 @@ -Subproject commit 4b4ec57b912744419d617614d0345f86bbcea99d +Subproject commit 7ebec7d6a4db230f53136c0a90d2cce46568eb2e diff --git a/nnf-dm b/nnf-dm index 3918377..42a3d3a 160000 --- a/nnf-dm +++ b/nnf-dm @@ -1 +1 @@ -Subproject commit 39183778f6daaaed73ddef77fa54670ff63f2ae9 +Subproject commit 42a3d3a4efd0e8389e5104d19ad2c668de1baff9 diff --git a/nnf-sos b/nnf-sos index dd36481..8749851 160000 --- a/nnf-sos +++ b/nnf-sos @@ -1 +1 @@ -Subproject commit dd36481958bf42161a7c489d1e8c00852a20b988 +Subproject commit 8749851dbc78c934d5f59302b3837a114a778fdd diff --git a/tools/kind.sh b/tools/kind.sh index 18f5acb..078ac80 100755 --- a/tools/kind.sh +++ b/tools/kind.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# Copyright 2021-2026 Hewlett Packard Enterprise Development LP # Other additional copyright holders may be indicated within. # # The entirety of this work is licensed under the Apache License, @@ -32,6 +32,51 @@ function clear_mock_fs { fi } +function inject_ca_certs { + local certs="$KIND_CA_CERTS" + local tmp_certs="" + + # If KIND_CA_CERTS is not set, auto-extract system CA certs. + if [[ -z "$certs" ]]; then + tmp_certs=$(mktemp /tmp/kind-ca-certs.XXXXXX.pem) + if [[ "$(uname)" == "Darwin" ]]; then + if [[ -f /Library/Keychains/System.keychain ]]; then + security find-certificate -a -p /Library/Keychains/System.keychain > "$tmp_certs" 2>/dev/null + fi + else + # Linux: copy the system CA bundle. + if [[ -f /etc/ssl/certs/ca-certificates.crt ]]; then + cp /etc/ssl/certs/ca-certificates.crt "$tmp_certs" + elif [[ -f /etc/pki/tls/certs/ca-bundle.crt ]]; then + cp /etc/pki/tls/certs/ca-bundle.crt "$tmp_certs" + fi + fi + + if [[ ! -s "$tmp_certs" ]]; then + echo "WARNING: Could not extract system CA certificates. Set KIND_CA_CERTS to a PEM file if image pulls fail with TLS errors." + rm -f "$tmp_certs" + return + fi + certs="$tmp_certs" + fi + + if [[ ! -f "$certs" ]]; then + echo "ERROR: KIND_CA_CERTS file not found: $certs" + return 1 + fi + + echo "Injecting CA certificates from $certs into KIND nodes..." + for node in $(kind get nodes); do + docker cp "$certs" "$node:/usr/local/share/ca-certificates/extra-ca-certs.crt" + docker exec "$node" update-ca-certificates + docker exec "$node" systemctl restart containerd + echo " $node: done" + done + + # Clean up temp file if we created one. + [[ -n "$tmp_certs" ]] && rm -f "$tmp_certs" +} + function create_cluster { CONFIG=kind-config.yaml @@ -92,6 +137,12 @@ EOF kind create cluster --wait 60s --image=kindest/node:v1.31.2 --config $CONFIG + # If corporate/custom CA certificates are available, inject them into + # each KIND node so containerd can pull from registries behind a TLS + # intercepting proxy. Set KIND_CA_CERTS to a PEM file path, or it + # defaults to /tmp/corporate-certs.pem. + inject_ca_certs + # Use the same init routines that we use on real hardware. # This applies taints and labels to rabbit nodes, and installs other # services that rabbit software requires.