Release v0.1.28

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+# NNF release reviewers — used by the release agent to assign PR reviewers.
+# All code owners are assigned as reviewers except the person creating the release.
+* @matthew-richerson @bdevcich @roehrich-hpe @ajfloeder

.github/agents/release.agent.md

@@ -0,0 +1,35 @@
+---
+description: "Use when performing an NNF software release, creating release branches, tagging releases, managing release PRs, running release-all.sh, comparing releases, or finalizing release notes. Covers releases across all NNF repositories."
+tools: [execute, read, edit, search, todo, web]
+argument-hint: "Specify the release type (patch, minor, or major) — defaults to patch"
+---
+
+You are the NNF Release Agent. Your job is to execute NNF software releases by following the documented release process precisely, step by step, with user approval at each gate.
+
+The entire release process is orchestrated by `release-all.sh` (located in `tools/release-all/`). It handles cloning, branching, PR creation, merging, and tagging for all NNF repos. The helper function `nnf_cmd` wraps `release-all.sh` with the correct flags.
+
+## Required Reading
+
+Before starting any release work, read these files completely:
+
+1. [Release Plan](../../tools/release-all/RELEASE-PLAN.md) — The authoritative, self-contained guide for releases (repo table, dependency chain, all phases)
+2. [Agent Instructions](../../tools/release-all/Instructions.md) — Error handling specifics, key gotchas, and guidance for updating the plan
+
+## Core Rules
+
+- **ALWAYS wait for explicit user approval before proceeding to the next step.** After every step, state what you did, whether it succeeded, and the evidence. **WAIT for the user.**
+- **ONE command per turn.** Run the command for ONE repo, show the full output, state PASS or FAIL with evidence, then STOP. Do not run the next repo's command until the user explicitly says to proceed. This applies even when the plan shows a `for` loop — execute the loop body manually, one repo per turn.
+- **ALWAYS run commands in a terminal** so the user can see them.
+- **ALWAYS append `2>&1 | cat`** to every `release-all.sh` and `gh` command — no exceptions.
+- **ONE repo at a time.** Execute one phase per repo sequentially. **NEVER parallelize** across repos.
+- **STOP on failure.** If a command fails, analyze the error and propose a fix. **Do NOT retry blindly.**
+- **NEVER use `--force`, `--no-verify`, or destructive operations** without asking the user first.
+- If you discover gaps or errors in the plan, **update RELEASE-PLAN.md** and tell the user what you changed.
+
+## Workflow
+
+1. Ask the user: What is the release type? (`patch`, `minor`, or `major` — default `patch`)
+2. Read the Release Plan and Agent Instructions completely.
+3. Follow the plan step by step, using the todo tool to track progress.
+4. At each approval gate, present your evidence and wait.
+5. After all repos are released, run `final-release-notes.sh` and `compare-releases.sh`.

cmd/main.go

@@ -104,6 +104,11 @@ func (cmd *DeployCmd) Run(ctx *Context) error {
 			return nil
 		}
 
+		// Wait for prerequisites that this module needs.
+		if err := waitForModulePrereqs(ctx, module); err != nil {
+			return err
+		}
+
 		if err := createSystemConfigFromSOS(ctx, system, module); err != nil {
 			return err
 		}
@@ -1044,6 +1049,100 @@ func getExampleOverlay(ctx *Context, system *config.System, module string) (stri
 	return "", nil
 }
 
+// Modules that require cert-manager webhook to be ready before deploying.
+var modulesCertManager = []string{
+	"lustre-fs-operator",
+	"dws",
+	"nnf-sos",
+}
+
+// Modules that require the DWS controller-manager to be ready before deploying,
+// because they use DWS CRDs that have a conversion webhook served by DWS.
+var modulesDWSController = []string{
+	"nnf-sos",
+	"nnf-dm",
+}
+
+// waitForModulePrereqs waits for prerequisite services that a module depends on.
+func waitForModulePrereqs(ctx *Context, module string) error {
+	if ctx.DryRun {
+		return nil
+	}
+
+	for _, m := range modulesCertManager {
+		if m == module {
+			if err := waitForCertManagerWebhook(ctx); err != nil {
+				return err
+			}
+			break
+		}
+	}
+
+	for _, m := range modulesDWSController {
+		if m == module {
+			if err := waitForDWSControllerManager(ctx); err != nil {
+				return err
+			}
+			break
+		}
+	}
+
+	return nil
+}
+
+// waitForCertManagerWebhook waits for the cert-manager webhook to be ready and
+// functional. Modules that create cert-manager Certificate or Issuer resources
+// need this webhook to be operational.
+func waitForCertManagerWebhook(ctx *Context) error {
+	fmt.Println("  Waiting for cert-manager webhook...")
+	cmd := exec.Command("kubectl", "wait", "deploy", "-n", "cert-manager",
+		"--timeout=180s", "cert-manager-webhook",
+		"--for", "jsonpath={.status.availableReplicas}=1")
+	if _, err := runCommand(ctx, cmd); err != nil {
+		return fmt.Errorf("cert-manager webhook deployment not ready: %w", err)
+	}
+
+	// The deployment being available doesn't guarantee the webhook is
+	// functional. Verify by attempting a server-side dry-run of an Issuer.
+	fmt.Println("  Verifying cert-manager webhook is functional...")
+	for attempts := 0; attempts < 60; attempts++ {
+		cmd = exec.Command("kubectl", "apply", "--dry-run=server", "-f", "-")
+		cmd.Stdin = strings.NewReader(`apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: deploy-dryrun-test
+  namespace: default
+spec:
+  selfSigned: {}
+`)
+		if out, err := cmd.CombinedOutput(); err == nil {
+			_ = out
+			break
+		}
+		if attempts == 59 {
+			return fmt.Errorf("cert-manager webhook not functional after 120s")
+		}
+		time.Sleep(2 * time.Second)
+	}
+
+	return nil
+}
+
+// waitForDWSControllerManager waits for the DWS controller-manager to be fully
+// ready. The DWS controller-manager serves the CRD conversion webhook needed by
+// modules that access DWS custom resources with multiple API versions.
+func waitForDWSControllerManager(ctx *Context) error {
+	fmt.Println("  Waiting for DWS controller-manager...")
+	cmd := exec.Command("kubectl", "wait", "deploy", "-n", "dws-system",
+		"--timeout=180s", "dws-controller-manager",
+		"--for", "jsonpath={.status.availableReplicas}=1")
+	if _, err := runCommand(ctx, cmd); err != nil {
+		return fmt.Errorf("DWS controller-manager not ready: %w", err)
+	}
+
+	return nil
+}
+
 func deployModule(ctx *Context, system *config.System, module string) error {
 
 	cmd := exec.Command("make", "deploy")

config/repositories.yaml

@@ -41,7 +41,7 @@ repositories:
     master: https://ghcr.io/nearnodeflash/lustre-fs-operator
     useRemoteK: false
     remoteReference:
-      build: v0.1.17
+      build: v0.1.18
       url: https://github.com/NearNodeFlash/lustre-fs-operator.git/config/default/?ref=%s
 buildConfiguration:
   # Environment variables to set when calling any 'make' or 'deploy'

tools/kind.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2021-2024 Hewlett Packard Enterprise Development LP
+# Copyright 2021-2026 Hewlett Packard Enterprise Development LP
 # Other additional copyright holders may be indicated within.
 #
 # The entirety of this work is licensed under the Apache License,
@@ -22,6 +22,42 @@
 set -e
 set -o pipefail
 
+usage() {
+    echo "Usage: $0 [--no-argocd] <CMD>"
+    echo
+    echo "Options:"
+    echo "  --no-argocd             Deploy without ArgoCD using the legacy overlay."
+    echo "                          Installs cert-manager, mpi-operator, and lustre"
+    echo "                          operators directly instead of via ArgoCD."
+    echo
+    echo "Commands:"
+    echo "  create                  Create a new KIND cluster."
+    echo "  destroy                 Destroy the existing KIND cluster."
+    echo "  reset                   Destroy the existing KIND cluster and create"
+    echo "                          a new cluster. Note: locally-built images are"
+    echo "                          not automatically reloaded -- re-run 'push' or"
+    echo "                          './nnf-deploy make kind-push' after a reset."
+    echo "  push                    Execute 'make kind-push' in each submodule dir."
+    echo "  argocd_attach <new_password>"
+    echo "                          Login to the argocd instance and set the new"
+    echo "                          password. Then add the Git repo to the instance."
+    echo "  argocd_login <new_password>"
+    echo "                          Only do the login to the argocd instance and set"
+    echo "                          the password."
+    echo "  argocd_add_git_repo     Only add the git repo to the argocd instance."
+    echo "  help                    Show this help message."
+}
+
+NO_ARGOCD=
+while [[ "$1" == --* ]]; do
+    case "$1" in
+    --no-argocd) NO_ARGOCD=1 ;;
+    --help) usage; exit 0 ;;
+    *) echo "Unknown option: $1"; exit 1 ;;
+    esac
+    shift
+done
+
 CMD="$1"
 shift
 
@@ -32,6 +68,67 @@ function clear_mock_fs {
     fi
 }
 
+function inject_ca_certs {
+    local certs="$KIND_CA_CERTS"
+    local tmp_certs=""
+
+    # If KIND_CA_CERTS is not set, auto-extract system CA certs.
+    if [[ -z "$certs" ]]; then
+        tmp_certs=$(mktemp /tmp/kind-ca-certs.XXXXXX.pem)
+        if [[ "$(uname)" == "Darwin" ]]; then
+            if [[ -f /Library/Keychains/System.keychain ]]; then
+                security find-certificate -a -p /Library/Keychains/System.keychain > "$tmp_certs" 2>/dev/null
+            fi
+        else
+            # Linux: copy the system CA bundle.
+            if [[ -f /etc/ssl/certs/ca-certificates.crt ]]; then
+                cp /etc/ssl/certs/ca-certificates.crt "$tmp_certs"
+            elif [[ -f /etc/pki/tls/certs/ca-bundle.crt ]]; then
+                cp /etc/pki/tls/certs/ca-bundle.crt "$tmp_certs"
+            fi
+        fi
+
+        if [[ ! -s "$tmp_certs" ]]; then
+            echo "WARNING: Could not extract system CA certificates. Set KIND_CA_CERTS to a PEM file if image pulls fail with TLS errors."
+            rm -f "$tmp_certs"
+            return
+        fi
+        certs="$tmp_certs"
+    fi
+
+    if [[ ! -f "$certs" ]]; then
+        echo "ERROR: KIND_CA_CERTS file not found: $certs"
+        return 1
+    fi
+
+    echo "Injecting CA certificates from $certs into KIND nodes..."
+    for node in $(kind get nodes); do
+        docker cp "$certs" "$node:/usr/local/share/ca-certificates/extra-ca-certs.crt"
+        docker exec "$node" update-ca-certificates
+        docker exec "$node" systemctl restart containerd
+        echo "  $node: done"
+    done
+
+    # Clean up temp file if we created one.
+    [[ -n "$tmp_certs" ]] && rm -f "$tmp_certs"
+}
+
+# Create the integration test user on KIND Rabbit worker nodes.
+# The nnf-integration-test suite verifies that UID/GID exist on Rabbit nodes
+# (defaults: UID=1051 GID=1052, the flux user). Without this, the BeforeSuite
+# check fails and no tests run.
+function create_test_user {
+    local uid=${NNF_USER_ID:-1051}
+    local gid=${NNF_GROUP_ID:-1052}
+
+    echo "Creating test user (UID=$uid, GID=$gid) on KIND Rabbit worker nodes..."
+    for node in $(kind get nodes | grep -v control-plane); do
+        # Create group and user if they don't already exist
+        docker exec "$node" sh -c "getent group $gid >/dev/null 2>&1 || groupadd -g $gid testgroup"
+        docker exec "$node" sh -c "getent passwd $uid >/dev/null 2>&1 || useradd -u $uid -g $gid -M -s /bin/bash testuser"
+    done
+}
+
 function create_cluster {
     CONFIG=kind-config.yaml
 
@@ -92,10 +189,23 @@ EOF
 
     kind create cluster --wait 60s --image=kindest/node:v1.31.2 --config $CONFIG
 
+    # If corporate/custom CA certificates are available, inject them into
+    # each KIND node so containerd can pull from registries behind a TLS
+    # intercepting proxy. Set KIND_CA_CERTS to a PEM file path, or it
+    # defaults to /tmp/corporate-certs.pem.
+    inject_ca_certs
+
     # Use the same init routines that we use on real hardware.
     # This applies taints and labels to rabbit nodes, and installs other
     # services that rabbit software requires.
+    if [[ -n $NO_ARGOCD ]]; then
+        echo "Deploying without ArgoCD (legacy overlay mode)..."
+        cp config/overlay-legacy.yaml-template overlay-legacy.yaml
+    fi
     ./nnf-deploy init
+
+    # Create the test user on Rabbit nodes for integration tests
+    create_test_user
 }
 
 function have_argocd {
@@ -180,23 +290,7 @@ function push_submodules {
     done
 }
 
-usage() {
-    echo "Usage: $0 <CMD>"
-    echo
-    echo "  create                  Create a new KIND cluster."
-    echo "  destroy                 Destroy the existing KIND cluster."
-    echo "  reset                   Destroy the existing KIND cluster and create"
-    echo "                          a new cluster."
-    echo "  push                    Execute 'make kind-push' in each submodule dir."
-    echo "  argocd_attach <new_password>"
-    echo "                          Login to the argocd instance and set the new"
-    echo "                          password. Then add the Git repo to the instance."
-    echo "  argocd_login <new_password>"
-    echo "                          Only do the login to the argocd instance and set"
-    echo "                          the password."
-    echo "  argocd_add_git_repo     Only add the git repo to the argocd instance."
 
-}
 
 if [[ "$CMD" == "create" ]]; then
     create_cluster
@@ -213,6 +307,8 @@ elif [[ "$CMD" == "argocd_login" ]]; then
     argocd_login "$*"
 elif [[ "$CMD" == "argocd_add_git_repo" ]]; then
     argocd_add_git_repo
+elif [[ "$CMD" == "help" || "$CMD" == "--help" ]]; then
+    usage
 else
     usage
     exit 1

tools/release-all/Instructions.md

@@ -0,0 +1,11 @@
+# Agent Instructions for NNF Software Release
+
+> This file supplements `RELEASE-PLAN.md` (the authoritative reference) with agent-specific guidance. **All procedural steps and gotchas are in the plan — do not duplicate them here.**
+
+## Updating the Plan
+
+If you encounter an issue not covered by RELEASE-PLAN.md, or if a step's success criteria are unclear:
+
+1. Resolve the issue with the user's guidance.
+2. Update RELEASE-PLAN.md with what you learned.
+3. Note the update to the user.