From a81dc843da46eed5e9d1a905b593b271965aa780 Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Mon, 16 Mar 2026 11:28:01 -0500
Subject: [PATCH 1/9] Fix compare-releases.sh: proper diff error handling, add
 options to for displaying the diffs

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 tools/release-all/compare-releases.sh | 52 +++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 6 deletions(-)

diff --git a/tools/release-all/compare-releases.sh b/tools/release-all/compare-releases.sh
index 47c797b..6989b69 100755
--- a/tools/release-all/compare-releases.sh
+++ b/tools/release-all/compare-releases.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2024 Hewlett Packard Enterprise Development LP
+# Copyright 2024-2026 Hewlett Packard Enterprise Development LP
 # Other additional copyright holders may be indicated within.
 #
 # The entirety of this work is licensed under the Apache License,
@@ -39,22 +39,32 @@ msg(){
 NNF_URL="git@github.com:NearNodeFlash/nnf-deploy.git"
 
 WORKSPACE="workingspace"
+DISPLAY_DIFF=false
+IMAGES_ONLY=false
 
 usage() {
     echo "Usage: $PROG [-h]"
-    echo "       $PROG [-w workspace_dir] [-u nnf_deploy_url] <VERSION1> <VERSION2>"
+    echo "       $PROG [-d] [-i] [-w workspace_dir] [-u nnf_deploy_url] <VERSION1> <VERSION2>"
     echo
     echo "  VERSION1          Starting version for comparison. This may be"
     echo "                    a tag such as 'v0.1.7'."
     echo "  VERSION2          Ending version for comparison. This may be"
     echo "                    a tag such as 'v0.1.8'."
+    echo "  -d                Display the full diff output."
+    echo "  -i                Display only image version changes."
     echo "  -w workspace_dir  Name for working directory. Default: '$WORKSPACE'"
     echo "  -u nnf_deploy_url URL to the nnf-deploy repo. Default: '$NNF_URL'"
     echo
 }
 
-while getopts "hw:u:" opt; do
+while getopts "dhiw:u:" opt; do
     case $opt in
+    d)
+        DISPLAY_DIFF=true
+        ;;
+    i)
+        IMAGES_ONLY=true
+        ;;
     u)
         NNF_URL="$OPTARG"
         ;;
@@ -106,7 +116,7 @@ prep_for_manifest() {
         rm -rf "$ver"
     fi
 
-    if ! gh release view "$ver" > /dev/null 2>&1; then
+    if ! gh release view -R "$NNF_URL" "$ver" > /dev/null 2>&1; then
         do_fail "Release $ver does not exist"
     fi
 }
@@ -116,10 +126,14 @@ fetch_and_unpack_manifest() {
     local vtar="$2"
 
     gh release download -R "$NNF_URL" -O "$vtar" -p manifests.tar "$ver" || do_fail "Unable to find manifests.tar for $ver"
+    [[ -s "$vtar" ]] || do_fail "Downloaded tarball $vtar is empty"
 
     mkdir "$ver" || do_fail "Cannot mkdir $ver"
     cd "$ver"
     tar xfo "../$vtar" || do_fail "Unable to extract tar $vtar"
+    local count
+    count=$(find . -type f | wc -l)
+    [[ $count -gt 0 ]] || do_fail "No files were extracted from $vtar"
     cd ..
 }
 
@@ -135,6 +149,32 @@ manifest "$ver1" "$v1tar"
 manifest "$ver2" "$v2tar"
 
 diff_file="manifest-$ver1-to-$ver2.diff"
-diff -uNr "$ver1" "$ver2" > "$diff_file" || echo
-msg "Manifest diffs for $ver1 to $ver2 are in $WORKSPACE/$diff_file"
+rc=0
+diff -uNr "$ver1" "$ver2" > "$diff_file" || rc=$?
+if [[ $rc -eq 0 ]]; then
+    msg "No differences found between $ver1 and $ver2"
+    rm -f "$diff_file"
+elif [[ $rc -eq 1 ]]; then
+    lines=$(wc -l < "$diff_file" | tr -d ' ')
+    changed_files=""
+    changed_files=$(diff -rq "$ver1" "$ver2") || true
+    changed=$(echo "$changed_files" | wc -l | tr -d ' ')
+    msg "Manifest diffs for $ver1 to $ver2 are in $WORKSPACE/$diff_file ($lines lines, $changed files changed)"
+    msg "Changed files:"
+    echo "$changed_files"
+
+    if [[ $IMAGES_ONLY == true ]]; then
+        msg ""
+        msg "Image version changes:"
+        grep -E '^[+-].*image:.*:' "$diff_file" | grep -v '^[+-][+-][+-]' || echo "  (no image changes found)"
+    fi
+
+    if [[ $DISPLAY_DIFF == true ]]; then
+        msg ""
+        msg "Full diff:"
+        cat "$diff_file"
+    fi
+else
+    do_fail "diff encountered an error (rc=$rc) comparing $ver1 and $ver2"
+fi
 

From 8f7fde1ec7e2abc507990d088e8e54704d888741 Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Mon, 16 Mar 2026 13:45:33 -0500
Subject: [PATCH 2/9] Improve KIND deployment reliability

KIND clusters fail to deploy out of the box due to three issues:
webhook race conditions, a removed container image, and missing
third-party services.

- Wait for cert-manager and DWS webhooks to be functional before
  deploying dependent modules (cmd/main.go)
- Update submodules to include gcr.io -> registry.k8s.io fix for
  kube-rbac-proxy
- Auto-inject system CA certificates into KIND nodes for corporate
  proxy environments (tools/kind.sh)
- Copy overlay-legacy.yaml-template so init installs cert-manager
  and mpi-operator without modifying shared config

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 cmd/main.go        | 99 ++++++++++++++++++++++++++++++++++++++++++++++
 dws                |  2 +-
 lustre-fs-operator |  2 +-
 nnf-dm             |  2 +-
 nnf-sos            |  2 +-
 tools/kind.sh      | 53 ++++++++++++++++++++++++-
 6 files changed, 155 insertions(+), 5 deletions(-)

diff --git a/cmd/main.go b/cmd/main.go
index 89e322f..ede6bee 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -104,6 +104,11 @@ func (cmd *DeployCmd) Run(ctx *Context) error {
 			return nil
 		}
 
+		// Wait for prerequisites that this module needs.
+		if err := waitForModulePrereqs(ctx, module); err != nil {
+			return err
+		}
+
 		if err := createSystemConfigFromSOS(ctx, system, module); err != nil {
 			return err
 		}
@@ -1044,6 +1049,100 @@ func getExampleOverlay(ctx *Context, system *config.System, module string) (stri
 	return "", nil
 }
 
+// Modules that require cert-manager webhook to be ready before deploying.
+var modulesCertManager = []string{
+	"lustre-fs-operator",
+	"dws",
+	"nnf-sos",
+}
+
+// Modules that require the DWS controller-manager to be ready before deploying,
+// because they use DWS CRDs that have a conversion webhook served by DWS.
+var modulesDWSController = []string{
+	"nnf-sos",
+	"nnf-dm",
+}
+
+// waitForModulePrereqs waits for prerequisite services that a module depends on.
+func waitForModulePrereqs(ctx *Context, module string) error {
+	if ctx.DryRun {
+		return nil
+	}
+
+	for _, m := range modulesCertManager {
+		if m == module {
+			if err := waitForCertManagerWebhook(ctx); err != nil {
+				return err
+			}
+			break
+		}
+	}
+
+	for _, m := range modulesDWSController {
+		if m == module {
+			if err := waitForDWSControllerManager(ctx); err != nil {
+				return err
+			}
+			break
+		}
+	}
+
+	return nil
+}
+
+// waitForCertManagerWebhook waits for the cert-manager webhook to be ready and
+// functional. Modules that create cert-manager Certificate or Issuer resources
+// need this webhook to be operational.
+func waitForCertManagerWebhook(ctx *Context) error {
+	fmt.Println("  Waiting for cert-manager webhook...")
+	cmd := exec.Command("kubectl", "wait", "deploy", "-n", "cert-manager",
+		"--timeout=180s", "cert-manager-webhook",
+		"--for", "jsonpath={.status.availableReplicas}=1")
+	if _, err := runCommand(ctx, cmd); err != nil {
+		return fmt.Errorf("cert-manager webhook deployment not ready: %w", err)
+	}
+
+	// The deployment being available doesn't guarantee the webhook is
+	// functional. Verify by attempting a server-side dry-run of an Issuer.
+	fmt.Println("  Verifying cert-manager webhook is functional...")
+	for attempts := 0; attempts < 60; attempts++ {
+		cmd = exec.Command("kubectl", "apply", "--dry-run=server", "-f", "-")
+		cmd.Stdin = strings.NewReader(`apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: deploy-dryrun-test
+  namespace: default
+spec:
+  selfSigned: {}
+`)
+		if out, err := cmd.CombinedOutput(); err == nil {
+			_ = out
+			break
+		}
+		if attempts == 59 {
+			return fmt.Errorf("cert-manager webhook not functional after 120s")
+		}
+		time.Sleep(2 * time.Second)
+	}
+
+	return nil
+}
+
+// waitForDWSControllerManager waits for the DWS controller-manager to be fully
+// ready. The DWS controller-manager serves the CRD conversion webhook needed by
+// modules that access DWS custom resources with multiple API versions.
+func waitForDWSControllerManager(ctx *Context) error {
+	fmt.Println("  Waiting for DWS controller-manager...")
+	cmd := exec.Command("kubectl", "wait", "deploy", "-n", "dws-system",
+		"--timeout=180s", "dws-controller-manager",
+		"--for", "jsonpath={.status.availableReplicas}=1")
+	if _, err := runCommand(ctx, cmd); err != nil {
+		return fmt.Errorf("DWS controller-manager not ready: %w", err)
+	}
+
+	return nil
+}
+
 func deployModule(ctx *Context, system *config.System, module string) error {
 
 	cmd := exec.Command("make", "deploy")
diff --git a/dws b/dws
index cf9255c..4e24ce7 160000
--- a/dws
+++ b/dws
@@ -1 +1 @@
-Subproject commit cf9255cfc6a772a6400dbc3ee172816d0e598a88
+Subproject commit 4e24ce7ae3727ff0c5cfca274c8c847cc84fdf05
diff --git a/lustre-fs-operator b/lustre-fs-operator
index 4b4ec57..7ebec7d 160000
--- a/lustre-fs-operator
+++ b/lustre-fs-operator
@@ -1 +1 @@
-Subproject commit 4b4ec57b912744419d617614d0345f86bbcea99d
+Subproject commit 7ebec7d6a4db230f53136c0a90d2cce46568eb2e
diff --git a/nnf-dm b/nnf-dm
index 3918377..42a3d3a 160000
--- a/nnf-dm
+++ b/nnf-dm
@@ -1 +1 @@
-Subproject commit 39183778f6daaaed73ddef77fa54670ff63f2ae9
+Subproject commit 42a3d3a4efd0e8389e5104d19ad2c668de1baff9
diff --git a/nnf-sos b/nnf-sos
index dd36481..8749851 160000
--- a/nnf-sos
+++ b/nnf-sos
@@ -1 +1 @@
-Subproject commit dd36481958bf42161a7c489d1e8c00852a20b988
+Subproject commit 8749851dbc78c934d5f59302b3837a114a778fdd
diff --git a/tools/kind.sh b/tools/kind.sh
index 18f5acb..078ac80 100755
--- a/tools/kind.sh
+++ b/tools/kind.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2021-2024 Hewlett Packard Enterprise Development LP
+# Copyright 2021-2026 Hewlett Packard Enterprise Development LP
 # Other additional copyright holders may be indicated within.
 #
 # The entirety of this work is licensed under the Apache License,
@@ -32,6 +32,51 @@ function clear_mock_fs {
     fi
 }
 
+function inject_ca_certs {
+    local certs="$KIND_CA_CERTS"
+    local tmp_certs=""
+
+    # If KIND_CA_CERTS is not set, auto-extract system CA certs.
+    if [[ -z "$certs" ]]; then
+        tmp_certs=$(mktemp /tmp/kind-ca-certs.XXXXXX.pem)
+        if [[ "$(uname)" == "Darwin" ]]; then
+            if [[ -f /Library/Keychains/System.keychain ]]; then
+                security find-certificate -a -p /Library/Keychains/System.keychain > "$tmp_certs" 2>/dev/null
+            fi
+        else
+            # Linux: copy the system CA bundle.
+            if [[ -f /etc/ssl/certs/ca-certificates.crt ]]; then
+                cp /etc/ssl/certs/ca-certificates.crt "$tmp_certs"
+            elif [[ -f /etc/pki/tls/certs/ca-bundle.crt ]]; then
+                cp /etc/pki/tls/certs/ca-bundle.crt "$tmp_certs"
+            fi
+        fi
+
+        if [[ ! -s "$tmp_certs" ]]; then
+            echo "WARNING: Could not extract system CA certificates. Set KIND_CA_CERTS to a PEM file if image pulls fail with TLS errors."
+            rm -f "$tmp_certs"
+            return
+        fi
+        certs="$tmp_certs"
+    fi
+
+    if [[ ! -f "$certs" ]]; then
+        echo "ERROR: KIND_CA_CERTS file not found: $certs"
+        return 1
+    fi
+
+    echo "Injecting CA certificates from $certs into KIND nodes..."
+    for node in $(kind get nodes); do
+        docker cp "$certs" "$node:/usr/local/share/ca-certificates/extra-ca-certs.crt"
+        docker exec "$node" update-ca-certificates
+        docker exec "$node" systemctl restart containerd
+        echo "  $node: done"
+    done
+
+    # Clean up temp file if we created one.
+    [[ -n "$tmp_certs" ]] && rm -f "$tmp_certs"
+}
+
 function create_cluster {
     CONFIG=kind-config.yaml
 
@@ -92,6 +137,12 @@ EOF
 
     kind create cluster --wait 60s --image=kindest/node:v1.31.2 --config $CONFIG
 
+    # If corporate/custom CA certificates are available, inject them into
+    # each KIND node so containerd can pull from registries behind a TLS
+    # intercepting proxy. Set KIND_CA_CERTS to a PEM file path, or it
+    # defaults to /tmp/corporate-certs.pem.
+    inject_ca_certs
+
     # Use the same init routines that we use on real hardware.
     # This applies taints and labels to rabbit nodes, and installs other
     # services that rabbit software requires.

From d98a807d1b0dcc635816557270f16a6b0ee9923a Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Tue, 17 Mar 2026 10:47:15 -0500
Subject: [PATCH 3/9] Add release agent documentation and helper script

Add VS Code custom agent (@release) and release documentation:
- .github/agents/release.agent.md: VSCode Custom agent for executing NNF releases
- tools/release-all/Instructions.md: Agent behavioral guidelines
- tools/release-all/RELEASE-PLAN.md: Authoritative release plan with vendoring,
  submodule pointer fixes, and dependency-ordered workflow
- tools/release-all/setup-release-env.sh: Helper script sourced to set up
  release environment variables and define nnf_cmd, nnf_create_pr,
  nnf_add_reviewers, nnf_gh_repo functions

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 .github/agents/release.agent.md        |  33 +++
 tools/release-all/Instructions.md      |  73 ++++++
 tools/release-all/RELEASE-PLAN.md      | 331 +++++++++++++++++++++++++
 tools/release-all/setup-release-env.sh | 127 ++++++++++
 4 files changed, 564 insertions(+)
 create mode 100644 .github/agents/release.agent.md
 create mode 100644 tools/release-all/Instructions.md
 create mode 100644 tools/release-all/RELEASE-PLAN.md
 create mode 100755 tools/release-all/setup-release-env.sh

diff --git a/.github/agents/release.agent.md b/.github/agents/release.agent.md
new file mode 100644
index 0000000..653ef42
--- /dev/null
+++ b/.github/agents/release.agent.md
@@ -0,0 +1,33 @@
+---
+description: "Use when performing an NNF software release, creating release branches, tagging releases, managing release PRs, running release-all.sh, comparing releases, or finalizing release notes. Covers releases across the 10 NNF repositories."
+tools: [execute, read, edit, search, todo, web]
+argument-hint: "Specify the release type (patch, minor, or major) — defaults to patch"
+---
+
+You are the NNF Release Agent. Your job is to execute NNF software releases by following the documented release process precisely, step by step, with user approval at each gate.
+
+## Required Reading
+
+Before starting any release work, read these files completely:
+
+1. [Release Plan](../../tools/release-all/RELEASE-PLAN.md) — The authoritative, self-contained guide for releases (repo table, dependency chain, all phases)
+2. [Agent Instructions](../../tools/release-all/Instructions.md) — Behavioral guidelines: execution model, approval gates, error handling, key gotchas
+
+## Core Rules
+
+- **Wait for user approval** after every step before proceeding.
+- After each step, state: what you did, whether it succeeded, and the evidence.
+- Run all commands in a terminal so the user can see them.
+- Append `2>&1 | cat` to all `release-all.sh` and `gh` commands.
+- Execute one phase per repo at a time — never parallelize across repos.
+- If a command fails, analyze the error and propose a fix. Do not retry blindly.
+- Never use `--force`, `--no-verify`, or destructive operations without asking.
+- If you discover gaps or errors in the plan, update RELEASE-PLAN.md and note the change.
+
+## Workflow
+
+1. Ask the user: What is the release type? (`patch`, `minor`, or `major` — default `patch`)
+2. Read the Release Plan and Agent Instructions completely.
+3. Follow the plan step by step, using the todo tool to track progress. The plan starts with cloning into a fresh working directory, then sources `setup-release-env.sh -B <type>` to set up environment variables and helper functions. Present the computed values to the user for confirmation.
+4. At each approval gate, present your evidence and wait.
+5. After all repos are released, run `final-release-notes.sh` and `compare-releases.sh`.
diff --git a/tools/release-all/Instructions.md b/tools/release-all/Instructions.md
new file mode 100644
index 0000000..a09fa48
--- /dev/null
+++ b/tools/release-all/Instructions.md
@@ -0,0 +1,73 @@
+# Agent Instructions for NNF Software Release
+
+> **Agent entry point:** In VS Code, invoke the `@release` agent from the chat picker (defined in `.github/agents/release.agent.md`). The agent automatically loads this file and `RELEASE-PLAN.md` as context.
+
+## Overview
+
+Execute the release plan defined in `RELEASE-PLAN.md` (in this same directory). That document is the authoritative, self-contained reference for the release process. Read it completely before starting.
+
+## Goal
+
+Create a release of the NNF software while ensuring the release process documented in `RELEASE-PLAN.md` is correct and repeatable. If you discover gaps, errors, or missing details in the plan, update it.
+
+## Agent Behavior
+
+### Execution Model
+
+- Run all commands in a terminal window so the user can see them.
+- Append `2>&1 | cat` to all `release-all.sh` and `gh` commands to avoid pager issues.
+- Execute one phase per repo at a time. Do not parallelize across repos.
+- Complete steps 4a–4d for each repo before moving to the next.
+
+### Approval Gates
+
+- **Wait for the user to approve** after each step before proceeding.
+- After each step, clearly state:
+  1. What you did
+  2. Whether you believe it succeeded
+  3. The evidence (command output, exit codes, API responses)
+- The user will respond with "approved", "approved. continue", or will coach you if something is wrong.
+
+### Error Handling
+
+- If a command fails, do **not** retry blindly. Analyze the error, explain it, and propose a fix.
+- If `merge-pr` produces no output, verify via: `gh api repos/<owner>/<repo>/pulls/<pr_number> 2>&1 | grep -o '"merged":[^,]*'`
+- If `tag-release` shows "Bypassed rule violations for refs/tags/..." — this is expected (tag protection rulesets allow admin/maintain bypass).
+- If a vendoring check fails, investigate the submodule pointer or vendor directory before proceeding.
+- Never use `--force`, `--no-verify`, or destructive operations without asking the user first.
+
+### Release Process
+
+The release process is the same regardless of how many repos have changes. The `release` phase automatically detects which repos have new commits since the last release. Repos with no changes report "No new changes to release" and are skipped in subsequent phases. Simply run all phases on all repos — the script handles the rest.
+
+### Key Gotchas (Learned from Experience)
+
+- `nnf_sos` requires the `-M` flag on ALL phases (not just `master`).
+- `nnf_mfu` and `nnf_ec` often have no changes — "No new changes to release" is normal, skip them.
+- `nnf_doc` must wait until after `nnf_deploy`'s GitHub Release is published (~60s after tagging). Verify with: `gh release view $NNF_RELEASE -R NearNodeFlash/nnf-deploy`
+- `nnf_doc` uses the `main` branch (not `master`). Its repo is `NearNodeFlash/NearNodeFlash.github.io`.
+- When a dependency changes (e.g., `nnf-sos`), all downstream repos that vendor it (`nnf-dm`, `nnf-integration-test`) must be revendored and their PRs merged **before** starting the release. If the vendoring check (Step 2) fails with "Peer modules are behind", follow Step 2a in RELEASE-PLAN.md to fix it. The check output prints the exact `go get` commands needed.
+- Always run vendoring checks on ALL repos — this catches stale submodule pointers.
+- After completing all releases, finalize release notes with `final-release-notes.sh` and run `compare-releases.sh` for verification.
+
+### Updating the Plan
+
+If you encounter an issue not covered by RELEASE-PLAN.md, or if a step's success criteria are unclear:
+
+1. Resolve the issue with the user's guidance.
+2. Update RELEASE-PLAN.md with what you learned.
+3. Note the update to the user.
+
+## Quick Start
+
+```sh
+# 1. Read the plan
+cat RELEASE-PLAN.md
+
+# 2. Ask the user: release type? (patch/minor/major, default patch)
+
+# 3. Source the helper: source ./setup-release-env.sh -B patch
+#    Confirm the printed summary with the user
+
+# 4. Follow RELEASE-PLAN.md step by step, waiting for approval at each gate.
+```
diff --git a/tools/release-all/RELEASE-PLAN.md b/tools/release-all/RELEASE-PLAN.md
new file mode 100644
index 0000000..bc47fd6
--- /dev/null
+++ b/tools/release-all/RELEASE-PLAN.md
@@ -0,0 +1,331 @@
+# NNF Software Release Plan
+
+This document is the authoritative, self-contained guide for executing an NNF software release. The release process is the same regardless of how many repos have changes — the `release` phase automatically detects which repos have new commits since the last release and skips those that don't. An agent or human following this document should be able to execute the entire release without external context.
+
+---
+
+## Repository Reference
+
+| Script Name | GitHub Repo | Owner | Default Branch | Notes |
+| --- | --- | --- | --- | --- |
+| `dws` | `DataWorkflowServices/dws` | DataWorkflowServices | `master` | |
+| `lustre_csi_driver` | `HewlettPackard/lustre-csi-driver` | HewlettPackard | `master` | |
+| `lustre_fs_operator` | `NearNodeFlash/lustre-fs-operator` | NearNodeFlash | `master` | |
+| `nnf_mfu` | `NearNodeFlash/nnf-mfu` | NearNodeFlash | `master` | Standalone, not a submodule |
+| `nnf_ec` | `NearNodeFlash/nnf-ec` | NearNodeFlash | `master` | Standalone, not a submodule |
+| `nnf_sos` | `NearNodeFlash/nnf-sos` | NearNodeFlash | `master` | Requires `-M` flag |
+| `nnf_dm` | `NearNodeFlash/nnf-dm` | NearNodeFlash | `master` | Vendors nnf-sos |
+| `nnf_integration_test` | `NearNodeFlash/nnf-integration-test` | NearNodeFlash | `master` | Vendors nnf-sos |
+| `nnf_deploy` | `NearNodeFlash/nnf-deploy` | NearNodeFlash | `master` | Has submodules for most repos |
+| `nnf_doc` | `NearNodeFlash/NearNodeFlash.github.io` | NearNodeFlash | **`main`** (not master) | Released last; tracks nnf-deploy version |
+
+**PR Reviewers:** Assign all members of the reviewer pool **except** the person creating the release:
+
+- `matthew-richerson`
+- `bdevcich`
+- `roehrich-hpe`
+- `ajfloeder`
+
+For example, if `ajfloeder` is running the release, the reviewers are `matthew-richerson`, `bdevcich`, `roehrich-hpe`.
+
+**Dependency chain (release order):**
+
+```text
+dws → lustre_csi_driver → lustre_fs_operator → nnf_mfu → nnf_ec → nnf_sos → nnf_dm → nnf_integration_test → nnf_deploy → nnf_doc
+```
+
+**Vendoring dependencies (peer modules in `go.mod`):**
+
+- `lustre_fs_operator` vendors `dws`
+- `nnf_sos` vendors `dws`, `lustre-fs-operator`, `nnf-ec`
+- `nnf_dm` vendors `dws`, `lustre-fs-operator`, `nnf-ec` (indirect), `nnf-sos`
+- `nnf_integration_test` vendors `dws`, `lustre-fs-operator`, `nnf-ec` (indirect), `nnf-sos`
+
+If an upstream repo changes, **all downstream repos that vendor it** may need revendoring before the release (see Step 2a). The vendoring check in Step 2 catches these automatically.
+
+---
+
+## Release Execution Plan
+
+### Prerequisites (Pre-flight Checklist)
+
+1. Ensure these tools are installed: `gh`, `yq` (Go version v4.x), `jq`, `perl`, `git`, `make`, `tput`, `sed`
+2. Set `GH_TOKEN` env var with a GitHub **classic** token (not fine-grained) with `repo` scope. The token is 40 characters, starting with `ghp_`.
+3. Verify SSH access to all repo URLs (`ssh -T git@github.com`)
+4. Decide the release type (`-B major|minor|patch`). **Always pass `-B` explicitly** — do not rely on the script default, which could change. Most releases use `-B patch`.
+5. **Pre-release vendoring check:** If any upstream repo has had changes merged to master since the last release that affect downstream vendoring, ensure downstream repos have been revendored **before** starting the release. For example, if `nnf-sos` changed, then `nnf-dm` and `nnf-integration-test` (which vendor `nnf-sos`) must have their vendor directories updated via PRs merged to master. Phase 1's vendoring checks will catch any mismatches.
+
+### Phase 0: Fresh Clone
+
+> The release uses a **volatile** working directory that is deleted and recreated at the start of each release. Set `$RELEASE_WORKDIR` to the desired location (e.g., `RELEASE_WORKDIR=~/release-work`).
+
+#### Step 0 — Clone nnf-deploy with submodules
+
+If `$RELEASE_WORKDIR` already exists, confirm with the user that it can be deleted. **If they say no, halt the release process.**
+
+```sh
+# Set the working directory for this release
+export RELEASE_WORKDIR=~/release-work   # adjust as needed
+
+# If $RELEASE_WORKDIR exists, ask the user before proceeding.
+# If they decline, stop here.
+rm -rf "$RELEASE_WORKDIR"
+mkdir -p "$RELEASE_WORKDIR"
+cd "$RELEASE_WORKDIR"
+git clone --recurse-submodules git@github.com:NearNodeFlash/nnf-deploy.git
+cd nnf-deploy
+```
+
+All subsequent steps run from `$RELEASE_WORKDIR/nnf-deploy`.
+
+### General Notes
+
+> **Helper functions:** After sourcing `setup-release-env.sh`, use `nnf_cmd <phase> <repo>` for all `release-all.sh` invocations. It automatically handles the `-B` flag, the `-M` flag for `nnf_sos`, and pipes through `cat` to avoid pager issues. Use `nnf_create_pr <repo>` for the `create-pr` phase (captures `$PR_NUMBER`) and `nnf_add_reviewers <repo>` to assign reviewers.
+>
+> **Manual invocations:** If you need to call `release-all.sh` directly, always pass `-B patch` (or `-B minor`/`-B major`) explicitly, add `-M` for `nnf_sos`, and append `2>&1 | cat`.
+
+### Phase 1: Discovery & Validation
+
+#### Step 1 — Set up and list repos
+
+```sh
+cd tools/release-all
+```
+
+Source the release environment helper to auto-compute versions, detect the release operator, and set reviewers:
+
+```sh
+source ./setup-release-env.sh -B patch   # or -B minor / -B major
+```
+
+This exports `$PREVIOUS_RELEASE`, `$NNF_RELEASE`, `$RELEASE_TYPE`, and `$RELEASE_REVIEWERS`, and defines helper functions `nnf_cmd`, `nnf_create_pr`, `nnf_add_reviewers`, and `nnf_gh_repo`. Confirm the printed summary with the user before proceeding.
+
+List repos:
+
+```sh
+./release-all.sh -L
+```
+
+Save the ordered list: `dws`, `lustre_csi_driver`, `lustre_fs_operator`, `nnf_mfu`, `nnf_ec`, `nnf_sos`, `nnf_dm`, `nnf_integration_test`, `nnf_deploy`, `nnf_doc`.
+
+#### Step 2 — Check vendoring *(sequential, per repo)*
+
+> Must be error-free before proceeding. This runs on **all 10 repos** including `nnf_doc` — vendoring checks validate the current state of master/main, which is independent of the release branching.
+
+```sh
+for repo in $(./release-all.sh -L); do
+    nnf_cmd master "$repo"
+done
+```
+
+> **Important:** Run vendoring checks on **all** repos, not just the ones you plan to release. This catches stale submodule pointers in `nnf-deploy` (e.g., a force-push on a submodule repo can leave the pointer at an orphaned commit SHA).
+
+#### Step 2a — Fix stale vendoring *(if Step 2 fails)*
+
+If the vendoring check for a repo reports **"Peer modules are behind"**, the error output includes the exact `go get` commands needed to update the vendor. Follow this procedure for each failing repo, working **in dependency order** per the vendoring table above (e.g., fix `lustre_fs_operator` before `nnf_sos`, fix `nnf_sos` before `nnf_dm` or `nnf_integration_test`).
+
+For each failing repo:
+
+1. **Read the error output.** It contains the `go get` command(s) with the specific module paths and branch targets.
+2. **Clone the repo** into a temporary directory using `nnf_gh_repo` to resolve the GitHub `owner/repo` path.
+3. **Create a branch** named `update-vendor`.
+4. **Run the `go get` commands** from the error output, then run `go mod tidy` and `go mod vendor`.
+5. **Commit** all changes with a signed-off commit (`-s`) and message `"Update vendor dependencies"`.
+6. **Push** the branch and **create a PR** titled `"Update vendor dependencies"` with body `"Pre-release vendoring update."`.
+7. **Assign reviewers** from `$RELEASE_REVIEWERS`.
+8. **Wait for CI** and reviewer approval, then **merge** the PR.
+9. **Verify the fix against origin:** Re-clone (repeat Step 0) so you're working from what's actually on GitHub, then re-run Step 2 starting from the repo you just fixed and continuing through the remaining downstream repos. The fixed repo must pass. If a downstream repo fails, repeat this procedure for it.
+10. Return to `$RELEASE_WORKDIR/nnf-deploy/tools/release-all`.
+
+> **Dependency order matters:** An upstream repo's vendoring PR must be merged to master before any downstream repo that vendors it can be fixed. For example, if both `nnf_sos` and `nnf_dm` fail, merge the `nnf_sos` fix first — `nnf_dm` vendors `nnf_sos` and needs the updated master.
+
+> **`nnf_deploy` submodule pointers:** Any submodule repo that has had commits merged to master since the last submodule pointer update — whether from development work or vendoring fixes — will cause `nnf_deploy`'s Step 2 check to fail with "Submodules are not up to date." The script catches this automatically. Fix it the same way: clone `nnf_deploy`, create a branch, run `git submodule update --remote` for the affected submodules, commit, PR, merge. Then re-clone and verify.
+
+After fixing and verifying the last failing repo, **re-run Step 2 on all repos** as a final safety net to confirm everything passes end to end.
+
+### Phase 2: Branch Creation
+
+#### Step 3 — Create trial release branches *(sequential, per repo)*
+
+Process all repos **except `nnf_doc`**, which is deferred (see below):
+
+```sh
+for repo in $(./release-all.sh -L | grep -v nnf_doc); do
+    nnf_cmd release "$repo"
+done
+```
+
+Review output for merge conflicts before continuing.
+
+> **Note:** `nnf_mfu` (and sometimes `nnf_ec`) may report "No new changes to release" if master has no commits since the last release. This is normal — skip the repo in subsequent phases.
+
+#### `nnf_doc` deferral
+
+`nnf_doc` must be deferred until after `nnf_deploy` is tagged and its GitHub Release is published. The `nnf_doc` script updates `mkdocs.yml` with the latest `nnf-deploy` release version by querying GitHub Releases (not just tags). Follow this sequence:
+
+1. Complete Steps 3–4d for all repos through `nnf_deploy`
+2. **Wait ~60 seconds** for `nnf_deploy`'s "Handle Release Tag" GitHub Actions workflow to publish the GitHub Release
+3. Verify: `gh release view $NNF_RELEASE -R NearNodeFlash/nnf-deploy 2>&1 | cat` should succeed
+4. Then run `nnf_doc` through Steps 3–4d:
+
+```sh
+nnf_cmd release nnf_doc
+nnf_cmd release-push nnf_doc
+nnf_create_pr nnf_doc
+nnf_add_reviewers nnf_doc
+nnf_cmd merge-pr nnf_doc
+nnf_cmd tag-release nnf_doc
+```
+
+> **Important:** `nnf_doc`'s repo is `NearNodeFlash/NearNodeFlash.github.io` and uses the `main` branch (not `master`).
+
+### Phase 3: Release Generation
+
+*Complete steps 4a–4d for each repo sequentially before moving on to the next. Present the results of all four sub-steps to the user and wait for approval once per repo (after step 4d), rather than after each individual sub-step.*
+
+> **Why sequential order matters:** Phase 2 creates release branches (not tags). Phase 3 then processes each repo through push → PR → merge → tag **in dependency order**. By the time a downstream repo (e.g., `nnf_dm`) reaches `create-pr`, any upstream repo it depends on (e.g., `nnf_sos`) is already tagged. The script's `release` phase handles submodule pointers to the correct release branches automatically.
+
+#### Step 4a — Push release branch
+
+*Only if no merge conflicts.*
+
+```sh
+nnf_cmd release-push <repo>
+```
+
+#### Step 4a-alt — Merge conflict resolution
+
+*Only if merge conflicts exist.* **Stop and present the conflict details to the user.** Merge conflicts during a release are rare and require human judgment. Do not attempt to resolve them automatically. Wait for the user to provide guidance before proceeding.
+
+#### Step 4b — Create PR
+
+```sh
+nnf_create_pr <repo>
+```
+
+The `create-pr` output includes the PR URL (e.g., `https://github.com/<owner>/<repo>/pull/123`). The `nnf_create_pr` function extracts the PR number and sets `$PR_NUMBER` automatically.
+
+> **Note:** The script does **not** assign PR reviewers automatically. After each `create-pr`, assign reviewers:
+
+```sh
+nnf_add_reviewers <repo>
+```
+
+#### Step 4c — Merge PR
+
+> Do NOT manually merge — let the tool do it.
+
+```sh
+nnf_cmd merge-pr <repo>
+```
+
+> **Note:** Before running `merge-pr`, ensure CI status checks on the PR have passed. If required checks are still running, `merge-pr` may fail.
+>
+> **Note:** `merge-pr` may produce no visible output even on success. Verify the merge completed:
+>
+> ```sh
+> gh api repos/<owner>/<repo>/pulls/<pr_number> 2>&1 | grep -o '"merged":[^,]*'
+> ```
+>
+> Expected output: `"merged":true`
+
+#### Step 4d — Tag the release
+
+*Creates the annotated tag required by CI/CD.*
+
+```sh
+nnf_cmd tag-release <repo>
+```
+
+> **Note:** `tag-release` output will include "Bypassed rule violations for refs/tags/...". This is expected. The repos have rulesets ("Auto-imported tag create protections" and "Auto-imported tag delete protections") that block creation, update, and deletion of `v*` tags by default — protecting release tags from unauthorized changes. Your account bypasses these rules because it has an admin or maintain role, which is in the ruleset's bypass list. The tags are created correctly.
+
+### Phase 4: Finalize
+
+#### Step 5 — Finalize release notes
+
+*Run only after ALL repos, including `nnf_doc`, are released. Run from `tools/release-all/`.*
+
+```sh
+./final-release-notes.sh -r $NNF_RELEASE 2>&1 | cat        # preview
+./final-release-notes.sh -r $NNF_RELEASE -C 2>&1 | cat      # commit
+```
+
+### Phase 5: Verification
+
+#### Step 6 — Compare release manifests
+
+*Run from `tools/release-all/`.*
+
+```sh
+./compare-releases.sh -i $PREVIOUS_RELEASE $NNF_RELEASE 2>&1 | cat
+```
+
+The `-i` flag displays image version changes inline. Use `-d` to display the full diff. The diff file is always saved to `workingspace/manifest-<ver1>-to-<ver2>.diff`.
+
+#### Step 7 — Verify GitHub releases
+
+Check each repo for correct tag, release notes, and artifacts (`manifests.tar` + `manifests-kind.tar` on `nnf-deploy`).
+
+---
+
+## Part B: Documentation Improvements & Corrections
+
+### Corrections (issues found by comparing docs to actual scripts)
+
+> Items marked ✅ have been applied to `release-all.md` on branch `improve-release-docs`.
+> Items marked 🔲 remain open.
+
+1. ✅ **Missing prerequisites** — Added full Prerequisites section with tool table, `GH_TOKEN` env var, and SSH access check.
+2. ✅ **Missing `-B` flag** — Documented `-B major|minor|patch` (default: `patch`) in the Steps section.
+3. 🔲 **Missing `-w`, `-M` flags** — `-x force-tag` is now documented for error recovery, but `-w` (workspace dir override) and `-M` (multi-API-version bypass) are not yet documented. These are advanced/optional and could be an enhancement.
+4. ✅ **Step structure was confusing** — Restructured as steps 3a–3d with bold headers; 3a branching path made explicit.
+5. ✅ **Annotated tag requirement not called out** — Added Important note at step 3d explaining annotated tag requirement and CI/CD rejection of lightweight tags.
+6. ✅ **`repo-list` output never shown** — Added the full 10-repo dependency-ordered list under step 0.
+7. ✅ **nnf-mfu and nnf-ec distinction unclear** — Added *(standalone repo, not a submodule)* annotations and dependency-order list with rationale.
+8. ✅ **No error recovery guidance** — Added `-x force-tag=vX.Y.Z` recovery, manual merge warning on step 3c.
+9. ✅ **`compare-releases.sh` not framed as verification** — Renamed section to "Verify the release" and framed as a recommended verification step.
+10. ✅ **`final-release-notes.sh` prereqs not listed** — Added note listing `yq`, `gh`, `jq`, `perl`, and `GH_TOKEN` requirement.
+11. ✅ **No mention of CI/CD workflow** — Added explanation of `handle_release_tag.yaml` at step 3d (creates GitHub release, attaches artifacts).
+12. ✅ **Step numbering was wrong** — Renumbered as 3a–3d (was 1–5 sub-list); removed phantom step "e".
+
+### Enhancements
+
+1. **Add a pre-flight checklist section** before Step 1
+2. **Add a troubleshooting section** — tag rejected, yq version mismatch, GH_TOKEN expired, merge conflicts
+3. **Add "What happens under the hood"** — Brief description of what each phase does internally (updates kustomization images, Helm versions, Dockerfile refs, repositories.yaml)
+4. **Document full repo ordering with rationale** — Explain the dependency chain (dws → lustre → nnf-mfu/ec → nnf-sos → nnf-dm → integration-test → deploy → docs)
+5. **Expand `tools/release-all/README.md`** — Add prerequisites and quick-start
+
+---
+
+### Relevant files
+
+- `tools/release-all/release-all.sh` — Main orchestration script, all phases
+- `tools/release-all/final-release-notes.sh` — Release notes finalization
+- `tools/release-all/compare-releases.sh` — Manifest diff tool
+- `tools/release-all/README.md` — Minimal, needs expansion
+- `.github/workflows/handle_release_tag.yaml` — CI/CD that creates GitHub releases on tag push
+- `config/repositories.yaml` — Submodule version tracking updated by release-push phase
+- External docs source: NearNodeFlash/NearNodeFlash.github.io repo
+
+### Decisions
+
+- Release execution follows the strict 10-repo dependency order — no parallelization
+- `nnf-mfu` and `nnf-ec` are released standalone (positions 4 & 5) despite not being submodules
+- `nnf_doc` released last, version should match nnf-deploy
+- Default `-B patch` unless breaking changes present
+
+### Verification
+
+1. `./release-all.sh -L` confirms 10 repos in dependency order
+2. Zero errors from each `master` phase before proceeding
+3. GitHub release page shows correct annotated tag after each `tag-release`
+4. `compare-releases.sh` diff validates expected changes
+5. `final-release-notes.sh` output includes all submodule notes + CRD API info
+6. nnf-deploy GitHub release has manifests.tar and manifests-kind.tar
+
+### Further Considerations
+
+1. **Release type** — Is this patch, minor, or major? Determines `-B` flag. Recommend: default `patch` unless breaking changes.
+2. **Doc improvements timing** — Corrections applied first (branch `improve-release-docs` in NearNodeFlash.github.io), to be used as a guide during the release. Enhancements can follow after the release.
+3. **README expansion** — Add prerequisites and quick-start to `tools/release-all/README.md` inline; keep detailed guide external.
diff --git a/tools/release-all/setup-release-env.sh b/tools/release-all/setup-release-env.sh
new file mode 100755
index 0000000..8bf6cb9
--- /dev/null
+++ b/tools/release-all/setup-release-env.sh
@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# setup-release-env.sh — Source this to set up the release environment.
+# Usage: source ./setup-release-env.sh [-B patch|minor|major]
+#
+# Exports: RELEASE_TYPE, PREVIOUS_RELEASE, NNF_RELEASE, RELEASE_USER,
+#          RELEASE_REVIEWERS
+# Defines: nnf_cmd, nnf_create_pr, nnf_add_reviewers, nnf_gh_repo
+
+# NOTE: This file is sourced, not executed. Do not use 'set -euo pipefail'
+# here — those options would persist in the caller's shell and cause any
+# non-zero exit (even transient) to kill the terminal session.
+
+# --- Parse -B flag -----------------------------------------------------------
+OPTIND=1
+RELEASE_TYPE="patch"
+while getopts "B:" opt; do
+    case $opt in
+        B) RELEASE_TYPE="$OPTARG" ;;
+        *) echo "Usage: source ./setup-release-env.sh [-B patch|minor|major]" >&2; return 1 ;;
+    esac
+done
+export RELEASE_TYPE
+
+# --- Compute versions --------------------------------------------------------
+export PREVIOUS_RELEASE
+PREVIOUS_RELEASE=$(gh release view --json tagName --jq '.tagName' \
+    -R NearNodeFlash/nnf-deploy 2>&1)
+if [[ -z "$PREVIOUS_RELEASE" || "$PREVIOUS_RELEASE" == *"release not found"* ]]; then
+    echo "ERROR: Could not determine previous release from NearNodeFlash/nnf-deploy" >&2
+    return 1
+fi
+
+case "$RELEASE_TYPE" in
+    major) PERL_CMD='($x=$F[0])=~s/^v//; printf("v%s", join(".", $x+1, 0, 0))' ;;
+    minor) PERL_CMD='print join(".", $F[0], $F[1]+1, 0)' ;;
+    patch) PERL_CMD='print join(".", $F[0], $F[1], $F[2]+1)' ;;
+    *)     echo "ERROR: Invalid release type '$RELEASE_TYPE'" >&2; return 1 ;;
+esac
+export NNF_RELEASE
+NNF_RELEASE=$(echo "$PREVIOUS_RELEASE" | perl -F'/\./' -ane "$PERL_CMD")
+
+# --- Detect operator & reviewers --------------------------------------------
+RELEASE_USER=$(gh api user --jq '.login' 2>&1)
+REVIEWER_POOL="matthew-richerson,bdevcich,roehrich-hpe,ajfloeder"
+export RELEASE_REVIEWERS
+RELEASE_REVIEWERS=$(echo "$REVIEWER_POOL" | tr ',' '\n' \
+    | grep -v "$RELEASE_USER" | paste -sd,)
+
+# --- Helper: run release-all.sh with auto -M and -B -------------------------
+# Usage: nnf_cmd <phase> <repo>
+nnf_cmd() {
+    local phase="$1" repo="$2"
+    local args=()
+    [[ "$repo" == "nnf_sos" ]] && args+=("-M")
+    ./release-all.sh -B "$RELEASE_TYPE" -P "$phase" -R "$repo" "${args[@]}" 2>&1 | cat
+}
+
+# --- Helper: create-pr + capture PR number -----------------------------------
+# Usage: nnf_create_pr <repo>
+#   Sets PR_NUMBER as a side effect.
+nnf_create_pr() {
+    local repo="$1"
+    local args=()
+    [[ "$repo" == "nnf_sos" ]] && args+=("-M")
+    local output
+    output=$(./release-all.sh -B "$RELEASE_TYPE" -P create-pr -R "$repo" "${args[@]}" 2>&1 | tee /dev/tty)
+    PR_NUMBER=$(echo "$output" | grep -o 'https://github.com/[^/ ]*/[^/ ]*/pull/[0-9]*' | awk -F'/' '{print $NF}' | head -n1)
+    export PR_NUMBER
+    echo "PR number: $PR_NUMBER"
+}
+
+# --- Helper: look up the GitHub <owner>/<repo> for a script name -------------
+# Usage: nnf_gh_repo <script_name>
+nnf_gh_repo() {
+    local repo="$1"
+    case "$repo" in
+        dws)                  echo "DataWorkflowServices/dws" ;;
+        lustre_csi_driver)    echo "HewlettPackard/lustre-csi-driver" ;;
+        lustre_fs_operator)   echo "NearNodeFlash/lustre-fs-operator" ;;
+        nnf_mfu)              echo "NearNodeFlash/nnf-mfu" ;;
+        nnf_ec)               echo "NearNodeFlash/nnf-ec" ;;
+        nnf_sos)              echo "NearNodeFlash/nnf-sos" ;;
+        nnf_dm)               echo "NearNodeFlash/nnf-dm" ;;
+        nnf_integration_test) echo "NearNodeFlash/nnf-integration-test" ;;
+        nnf_deploy)           echo "NearNodeFlash/nnf-deploy" ;;
+        nnf_doc)              echo "NearNodeFlash/NearNodeFlash.github.io" ;;
+        *)                    echo "UNKNOWN/$repo" ;;
+    esac
+}
+
+# --- Helper: assign reviewers to the current PR ------------------------------
+# Usage: nnf_add_reviewers <repo>
+#   Requires PR_NUMBER to be set (via nnf_create_pr).
+nnf_add_reviewers() {
+    local repo="$1"
+    local gh_repo
+    gh_repo=$(nnf_gh_repo "$repo")
+    gh pr edit "$PR_NUMBER" --add-reviewer "$RELEASE_REVIEWERS" \
+        -R "$gh_repo" 2>&1 | cat
+}
+
+# --- Summary -----------------------------------------------------------------
+echo "──────────────────────────────────────"
+echo "  Previous release : $PREVIOUS_RELEASE"
+echo "  New release      : $NNF_RELEASE"
+echo "  Release type     : $RELEASE_TYPE"
+echo "  Operator         : $RELEASE_USER"
+echo "  Reviewers        : $RELEASE_REVIEWERS"
+echo "──────────────────────────────────────"

From df81fe4cb6a27f33a574494676708e650fae7d11 Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Wed, 18 Mar 2026 11:09:33 -0500
Subject: [PATCH 4/9] Address PR review feedback and dry-run findings

- Deduplicate Instructions.md (remove sections that duplicate release.agent.md)
- Strengthen core rules in release.agent.md with NEVER/ALWAYS/STOP emphasis
- Name release-all.sh as the central tool in agent file and plan intro
- Add .github/CODEOWNERS with 4 reviewers
- Read CODEOWNERS in setup-release-env.sh instead of hardcoded pool
- Add make clean-bin loop before rm -rf in Step 0
- Add gh auth status pre-flight check (prereq 3)
- Add retry guidance for nnf_deploy GitHub Release wait
- Add empty RELEASE_REVIEWERS warning guard

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 .github/CODEOWNERS                     |  3 ++
 .github/agents/release.agent.md        | 21 ++++----
 tools/release-all/Instructions.md      | 66 +-------------------------
 tools/release-all/RELEASE-PLAN.md      | 62 +++++++-----------------
 tools/release-all/setup-release-env.sh | 12 ++++-
 5 files changed, 45 insertions(+), 119 deletions(-)
 create mode 100644 .github/CODEOWNERS

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 0000000..625e1a5
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,3 @@
+# NNF release reviewers — used by the release agent to assign PR reviewers.
+# All code owners are assigned as reviewers except the person creating the release.
+* @matthew-richerson @bdevcich @roehrich-hpe @ajfloeder
diff --git a/.github/agents/release.agent.md b/.github/agents/release.agent.md
index 653ef42..a09b8f4 100644
--- a/.github/agents/release.agent.md
+++ b/.github/agents/release.agent.md
@@ -6,28 +6,29 @@ argument-hint: "Specify the release type (patch, minor, or major) — defaults t
 
 You are the NNF Release Agent. Your job is to execute NNF software releases by following the documented release process precisely, step by step, with user approval at each gate.
 
+The entire release process is orchestrated by `release-all.sh` (located in `tools/release-all/`). It handles cloning, branching, PR creation, merging, and tagging for all NNF repos. The helper function `nnf_cmd` wraps `release-all.sh` with the correct flags.
+
 ## Required Reading
 
 Before starting any release work, read these files completely:
 
 1. [Release Plan](../../tools/release-all/RELEASE-PLAN.md) — The authoritative, self-contained guide for releases (repo table, dependency chain, all phases)
-2. [Agent Instructions](../../tools/release-all/Instructions.md) — Behavioral guidelines: execution model, approval gates, error handling, key gotchas
+2. [Agent Instructions](../../tools/release-all/Instructions.md) — Error handling specifics, key gotchas, and guidance for updating the plan
 
 ## Core Rules
 
-- **Wait for user approval** after every step before proceeding.
-- After each step, state: what you did, whether it succeeded, and the evidence.
-- Run all commands in a terminal so the user can see them.
-- Append `2>&1 | cat` to all `release-all.sh` and `gh` commands.
-- Execute one phase per repo at a time — never parallelize across repos.
-- If a command fails, analyze the error and propose a fix. Do not retry blindly.
-- Never use `--force`, `--no-verify`, or destructive operations without asking.
-- If you discover gaps or errors in the plan, update RELEASE-PLAN.md and note the change.
+- **ALWAYS wait for explicit user approval before proceeding to the next step.** After every step, state what you did, whether it succeeded, and the evidence. **WAIT for the user.**
+- **ALWAYS run commands in a terminal** so the user can see them.
+- **ALWAYS append `2>&1 | cat`** to every `release-all.sh` and `gh` command — no exceptions.
+- **ONE repo at a time.** Execute one phase per repo sequentially. **NEVER parallelize** across repos.
+- **STOP on failure.** If a command fails, analyze the error and propose a fix. **Do NOT retry blindly.**
+- **NEVER use `--force`, `--no-verify`, or destructive operations** without asking the user first.
+- If you discover gaps or errors in the plan, **update RELEASE-PLAN.md** and tell the user what you changed.
 
 ## Workflow
 
 1. Ask the user: What is the release type? (`patch`, `minor`, or `major` — default `patch`)
 2. Read the Release Plan and Agent Instructions completely.
-3. Follow the plan step by step, using the todo tool to track progress. The plan starts with cloning into a fresh working directory, then sources `setup-release-env.sh -B <type>` to set up environment variables and helper functions. Present the computed values to the user for confirmation.
+3. Follow the plan step by step, using the todo tool to track progress.
 4. At each approval gate, present your evidence and wait.
 5. After all repos are released, run `final-release-notes.sh` and `compare-releases.sh`.
diff --git a/tools/release-all/Instructions.md b/tools/release-all/Instructions.md
index a09fa48..a888799 100644
--- a/tools/release-all/Instructions.md
+++ b/tools/release-all/Instructions.md
@@ -1,73 +1,11 @@
 # Agent Instructions for NNF Software Release
 
-> **Agent entry point:** In VS Code, invoke the `@release` agent from the chat picker (defined in `.github/agents/release.agent.md`). The agent automatically loads this file and `RELEASE-PLAN.md` as context.
+> This file supplements `RELEASE-PLAN.md` (the authoritative reference) with agent-specific guidance. **All procedural steps and gotchas are in the plan — do not duplicate them here.**
 
-## Overview
-
-Execute the release plan defined in `RELEASE-PLAN.md` (in this same directory). That document is the authoritative, self-contained reference for the release process. Read it completely before starting.
-
-## Goal
-
-Create a release of the NNF software while ensuring the release process documented in `RELEASE-PLAN.md` is correct and repeatable. If you discover gaps, errors, or missing details in the plan, update it.
-
-## Agent Behavior
-
-### Execution Model
-
-- Run all commands in a terminal window so the user can see them.
-- Append `2>&1 | cat` to all `release-all.sh` and `gh` commands to avoid pager issues.
-- Execute one phase per repo at a time. Do not parallelize across repos.
-- Complete steps 4a–4d for each repo before moving to the next.
-
-### Approval Gates
-
-- **Wait for the user to approve** after each step before proceeding.
-- After each step, clearly state:
-  1. What you did
-  2. Whether you believe it succeeded
-  3. The evidence (command output, exit codes, API responses)
-- The user will respond with "approved", "approved. continue", or will coach you if something is wrong.
-
-### Error Handling
-
-- If a command fails, do **not** retry blindly. Analyze the error, explain it, and propose a fix.
-- If `merge-pr` produces no output, verify via: `gh api repos/<owner>/<repo>/pulls/<pr_number> 2>&1 | grep -o '"merged":[^,]*'`
-- If `tag-release` shows "Bypassed rule violations for refs/tags/..." — this is expected (tag protection rulesets allow admin/maintain bypass).
-- If a vendoring check fails, investigate the submodule pointer or vendor directory before proceeding.
-- Never use `--force`, `--no-verify`, or destructive operations without asking the user first.
-
-### Release Process
-
-The release process is the same regardless of how many repos have changes. The `release` phase automatically detects which repos have new commits since the last release. Repos with no changes report "No new changes to release" and are skipped in subsequent phases. Simply run all phases on all repos — the script handles the rest.
-
-### Key Gotchas (Learned from Experience)
-
-- `nnf_sos` requires the `-M` flag on ALL phases (not just `master`).
-- `nnf_mfu` and `nnf_ec` often have no changes — "No new changes to release" is normal, skip them.
-- `nnf_doc` must wait until after `nnf_deploy`'s GitHub Release is published (~60s after tagging). Verify with: `gh release view $NNF_RELEASE -R NearNodeFlash/nnf-deploy`
-- `nnf_doc` uses the `main` branch (not `master`). Its repo is `NearNodeFlash/NearNodeFlash.github.io`.
-- When a dependency changes (e.g., `nnf-sos`), all downstream repos that vendor it (`nnf-dm`, `nnf-integration-test`) must be revendored and their PRs merged **before** starting the release. If the vendoring check (Step 2) fails with "Peer modules are behind", follow Step 2a in RELEASE-PLAN.md to fix it. The check output prints the exact `go get` commands needed.
-- Always run vendoring checks on ALL repos — this catches stale submodule pointers.
-- After completing all releases, finalize release notes with `final-release-notes.sh` and run `compare-releases.sh` for verification.
-
-### Updating the Plan
+## Updating the Plan
 
 If you encounter an issue not covered by RELEASE-PLAN.md, or if a step's success criteria are unclear:
 
 1. Resolve the issue with the user's guidance.
 2. Update RELEASE-PLAN.md with what you learned.
 3. Note the update to the user.
-
-## Quick Start
-
-```sh
-# 1. Read the plan
-cat RELEASE-PLAN.md
-
-# 2. Ask the user: release type? (patch/minor/major, default patch)
-
-# 3. Source the helper: source ./setup-release-env.sh -B patch
-#    Confirm the printed summary with the user
-
-# 4. Follow RELEASE-PLAN.md step by step, waiting for approval at each gate.
-```
diff --git a/tools/release-all/RELEASE-PLAN.md b/tools/release-all/RELEASE-PLAN.md
index bc47fd6..82d9e51 100644
--- a/tools/release-all/RELEASE-PLAN.md
+++ b/tools/release-all/RELEASE-PLAN.md
@@ -1,6 +1,6 @@
 # NNF Software Release Plan
 
-This document is the authoritative, self-contained guide for executing an NNF software release. The release process is the same regardless of how many repos have changes — the `release` phase automatically detects which repos have new commits since the last release and skips those that don't. An agent or human following this document should be able to execute the entire release without external context.
+This document is the authoritative, self-contained guide for executing an NNF software release. All release phases are driven by `release-all.sh` in this directory, wrapped by the `nnf_cmd` helper function. The release process is the same regardless of how many repos have changes — the `release` phase automatically detects which repos have new commits since the last release and skips those that don't. An agent or human following this document should be able to execute the entire release without external context.
 
 ---
 
@@ -19,14 +19,7 @@ This document is the authoritative, self-contained guide for executing an NNF so
 | `nnf_deploy` | `NearNodeFlash/nnf-deploy` | NearNodeFlash | `master` | Has submodules for most repos |
 | `nnf_doc` | `NearNodeFlash/NearNodeFlash.github.io` | NearNodeFlash | **`main`** (not master) | Released last; tracks nnf-deploy version |
 
-**PR Reviewers:** Assign all members of the reviewer pool **except** the person creating the release:
-
-- `matthew-richerson`
-- `bdevcich`
-- `roehrich-hpe`
-- `ajfloeder`
-
-For example, if `ajfloeder` is running the release, the reviewers are `matthew-richerson`, `bdevcich`, `roehrich-hpe`.
+**PR Reviewers:** Defined in `.github/CODEOWNERS`. Assign all code owners **except** the person creating the release. The `setup-release-env.sh` helper reads CODEOWNERS automatically and sets `$RELEASE_REVIEWERS`.
 
 **Dependency chain (release order):**
 
@@ -51,9 +44,10 @@ If an upstream repo changes, **all downstream repos that vendor it** may need re
 
 1. Ensure these tools are installed: `gh`, `yq` (Go version v4.x), `jq`, `perl`, `git`, `make`, `tput`, `sed`
 2. Set `GH_TOKEN` env var with a GitHub **classic** token (not fine-grained) with `repo` scope. The token is 40 characters, starting with `ghp_`.
-3. Verify SSH access to all repo URLs (`ssh -T git@github.com`)
-4. Decide the release type (`-B major|minor|patch`). **Always pass `-B` explicitly** — do not rely on the script default, which could change. Most releases use `-B patch`.
-5. **Pre-release vendoring check:** If any upstream repo has had changes merged to master since the last release that affect downstream vendoring, ensure downstream repos have been revendored **before** starting the release. For example, if `nnf-sos` changed, then `nnf-dm` and `nnf-integration-test` (which vendor `nnf-sos`) must have their vendor directories updated via PRs merged to master. Phase 1's vendoring checks will catch any mismatches.
+3. Verify `gh` authentication: `gh auth status`. Confirm it shows the expected user and token scopes.
+4. Verify SSH access to all repo URLs (`ssh -T git@github.com`)
+5. Decide the release type (`-B major|minor|patch`). **Always pass `-B` explicitly** — do not rely on the script default, which could change. Most releases use `-B patch`.
+6. **Pre-release vendoring check:** If any upstream repo has had changes merged to master since the last release that affect downstream vendoring, ensure downstream repos have been revendored **before** starting the release. For example, if `nnf-sos` changed, then `nnf-dm` and `nnf-integration-test` (which vendor `nnf-sos`) must have their vendor directories updated via PRs merged to master. Phase 1's vendoring checks will catch any mismatches.
 
 ### Phase 0: Fresh Clone
 
@@ -69,6 +63,14 @@ export RELEASE_WORKDIR=~/release-work   # adjust as needed
 
 # If $RELEASE_WORKDIR exists, ask the user before proceeding.
 # If they decline, stop here.
+
+# Clean build artifacts before removing (rm -rf can fail on build outputs)
+if [ -d "$RELEASE_WORKDIR/nnf-deploy/workingspace" ]; then
+    for repo in "$RELEASE_WORKDIR"/nnf-deploy/workingspace/*/; do
+        ( cd "$repo" && make clean-bin 2>/dev/null || true )
+    done
+fi
+
 rm -rf "$RELEASE_WORKDIR"
 mkdir -p "$RELEASE_WORKDIR"
 cd "$RELEASE_WORKDIR"
@@ -134,11 +136,11 @@ For each failing repo:
 6. **Push** the branch and **create a PR** titled `"Update vendor dependencies"` with body `"Pre-release vendoring update."`.
 7. **Assign reviewers** from `$RELEASE_REVIEWERS`.
 8. **Wait for CI** and reviewer approval, then **merge** the PR.
-9. **Verify the fix against origin:** Re-clone (repeat Step 0) so you're working from what's actually on GitHub, then re-run Step 2 starting from the repo you just fixed and continuing through the remaining downstream repos. The fixed repo must pass. If a downstream repo fails, repeat this procedure for it.
+9. **Verify the fix against origin:** Re-clone (repeat Step 0) so you're working from what's actually on GitHub, then re-run the `nnf_cmd master <repo>` check starting from the repo you just fixed and continuing through all remaining downstream repos in order. The fixed repo must now pass. If a downstream repo fails, repeat this procedure (steps 1–9) for it.
 10. Return to `$RELEASE_WORKDIR/nnf-deploy/tools/release-all`.
 
 > **Dependency order matters:** An upstream repo's vendoring PR must be merged to master before any downstream repo that vendors it can be fixed. For example, if both `nnf_sos` and `nnf_dm` fail, merge the `nnf_sos` fix first — `nnf_dm` vendors `nnf_sos` and needs the updated master.
-
+>
 > **`nnf_deploy` submodule pointers:** Any submodule repo that has had commits merged to master since the last submodule pointer update — whether from development work or vendoring fixes — will cause `nnf_deploy`'s Step 2 check to fail with "Submodules are not up to date." The script catches this automatically. Fix it the same way: clone `nnf_deploy`, create a branch, run `git submodule update --remote` for the affected submodules, commit, PR, merge. Then re-clone and verify.
 
 After fixing and verifying the last failing repo, **re-run Step 2 on all repos** as a final safety net to confirm everything passes end to end.
@@ -157,7 +159,7 @@ done
 
 Review output for merge conflicts before continuing.
 
-> **Note:** `nnf_mfu` (and sometimes `nnf_ec`) may report "No new changes to release" if master has no commits since the last release. This is normal — skip the repo in subsequent phases.
+> **Note:** `nnf_mfu` and `nnf_ec` are standalone repos that often have no new commits between releases. If either reports "No new changes to release", this is normal — **skip that repo in all subsequent phases** (Steps 4a–4d and the `nnf_doc` sequence). Do not attempt to push, PR, merge, or tag a repo that had no changes.
 
 #### `nnf_doc` deferral
 
@@ -165,7 +167,7 @@ Review output for merge conflicts before continuing.
 
 1. Complete Steps 3–4d for all repos through `nnf_deploy`
 2. **Wait ~60 seconds** for `nnf_deploy`'s "Handle Release Tag" GitHub Actions workflow to publish the GitHub Release
-3. Verify: `gh release view $NNF_RELEASE -R NearNodeFlash/nnf-deploy 2>&1 | cat` should succeed
+3. Verify: `gh release view $NNF_RELEASE -R NearNodeFlash/nnf-deploy 2>&1 | cat` should succeed. If it fails with "release not found", wait another 30 seconds and retry. The workflow typically completes within 2 minutes; if it hasn't after 5 minutes, check the Actions tab on the `nnf-deploy` repo for errors.
 4. Then run `nnf_doc` through Steps 3–4d:
 
 ```sh
@@ -266,35 +268,7 @@ The `-i` flag displays image version changes inline. Use `-d` to display the ful
 
 Check each repo for correct tag, release notes, and artifacts (`manifests.tar` + `manifests-kind.tar` on `nnf-deploy`).
 
----
 
-## Part B: Documentation Improvements & Corrections
-
-### Corrections (issues found by comparing docs to actual scripts)
-
-> Items marked ✅ have been applied to `release-all.md` on branch `improve-release-docs`.
-> Items marked 🔲 remain open.
-
-1. ✅ **Missing prerequisites** — Added full Prerequisites section with tool table, `GH_TOKEN` env var, and SSH access check.
-2. ✅ **Missing `-B` flag** — Documented `-B major|minor|patch` (default: `patch`) in the Steps section.
-3. 🔲 **Missing `-w`, `-M` flags** — `-x force-tag` is now documented for error recovery, but `-w` (workspace dir override) and `-M` (multi-API-version bypass) are not yet documented. These are advanced/optional and could be an enhancement.
-4. ✅ **Step structure was confusing** — Restructured as steps 3a–3d with bold headers; 3a branching path made explicit.
-5. ✅ **Annotated tag requirement not called out** — Added Important note at step 3d explaining annotated tag requirement and CI/CD rejection of lightweight tags.
-6. ✅ **`repo-list` output never shown** — Added the full 10-repo dependency-ordered list under step 0.
-7. ✅ **nnf-mfu and nnf-ec distinction unclear** — Added *(standalone repo, not a submodule)* annotations and dependency-order list with rationale.
-8. ✅ **No error recovery guidance** — Added `-x force-tag=vX.Y.Z` recovery, manual merge warning on step 3c.
-9. ✅ **`compare-releases.sh` not framed as verification** — Renamed section to "Verify the release" and framed as a recommended verification step.
-10. ✅ **`final-release-notes.sh` prereqs not listed** — Added note listing `yq`, `gh`, `jq`, `perl`, and `GH_TOKEN` requirement.
-11. ✅ **No mention of CI/CD workflow** — Added explanation of `handle_release_tag.yaml` at step 3d (creates GitHub release, attaches artifacts).
-12. ✅ **Step numbering was wrong** — Renumbered as 3a–3d (was 1–5 sub-list); removed phantom step "e".
-
-### Enhancements
-
-1. **Add a pre-flight checklist section** before Step 1
-2. **Add a troubleshooting section** — tag rejected, yq version mismatch, GH_TOKEN expired, merge conflicts
-3. **Add "What happens under the hood"** — Brief description of what each phase does internally (updates kustomization images, Helm versions, Dockerfile refs, repositories.yaml)
-4. **Document full repo ordering with rationale** — Explain the dependency chain (dws → lustre → nnf-mfu/ec → nnf-sos → nnf-dm → integration-test → deploy → docs)
-5. **Expand `tools/release-all/README.md`** — Add prerequisites and quick-start
 
 ---
 
diff --git a/tools/release-all/setup-release-env.sh b/tools/release-all/setup-release-env.sh
index 8bf6cb9..8198f8c 100755
--- a/tools/release-all/setup-release-env.sh
+++ b/tools/release-all/setup-release-env.sh
@@ -59,11 +59,21 @@ NNF_RELEASE=$(echo "$PREVIOUS_RELEASE" | perl -F'/\./' -ane "$PERL_CMD")
 
 # --- Detect operator & reviewers --------------------------------------------
 RELEASE_USER=$(gh api user --jq '.login' 2>&1)
-REVIEWER_POOL="matthew-richerson,bdevcich,roehrich-hpe,ajfloeder"
+CODEOWNERS_FILE="../../.github/CODEOWNERS"
+if [[ ! -f "$CODEOWNERS_FILE" ]]; then
+    echo "ERROR: $CODEOWNERS_FILE not found. Run from tools/release-all/ inside an nnf-deploy clone." >&2
+    return 1
+fi
+REVIEWER_POOL=$(grep '^\*' "$CODEOWNERS_FILE" | tr -s ' ' | cut -d' ' -f2- | tr ' ' '\n' | sed 's/@//' | paste -sd,)
 export RELEASE_REVIEWERS
 RELEASE_REVIEWERS=$(echo "$REVIEWER_POOL" | tr ',' '\n' \
     | grep -v "$RELEASE_USER" | paste -sd,)
 
+if [[ -z "$RELEASE_REVIEWERS" ]]; then
+    echo "WARNING: RELEASE_REVIEWERS is empty — you are the only code owner in CODEOWNERS." >&2
+    echo "         You will need to assign reviewers manually for each PR." >&2
+fi
+
 # --- Helper: run release-all.sh with auto -M and -B -------------------------
 # Usage: nnf_cmd <phase> <repo>
 nnf_cmd() {

From bbf19a248a6d4bb33dce5e577230c6ef9be22388 Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Mon, 23 Mar 2026 08:59:48 -0500
Subject: [PATCH 5/9] Add nnf-storedversions-maint to release process

- Add nnf_storedversions_maint to release-all.sh repo map (index 5,
  between nnf_ec and nnf_sos; uses main branch)
- Add config/manager kustomization update for nnf_storedversions_maint
  in update_own_release_references()
- Add nnf_storedversions_maint to nnf_gh_repo() in setup-release-env.sh
- Update RELEASE-PLAN.md: repo table, dependency chain, vendoring note,
  no-changes note, saved repo list
- Update final-release-notes.sh to include nnf_storedversions_maint as
  a standalone block (between nnf_deploy and nnf_doc)

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 .github/agents/release.agent.md          |  2 +-
 tools/release-all/RELEASE-PLAN.md        |  8 ++++---
 tools/release-all/final-release-notes.sh |  6 +++++
 tools/release-all/release-all.sh         | 28 +++++++++++++++---------
 tools/release-all/setup-release-env.sh   |  1 +
 5 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/.github/agents/release.agent.md b/.github/agents/release.agent.md
index a09b8f4..b60af6a 100644
--- a/.github/agents/release.agent.md
+++ b/.github/agents/release.agent.md
@@ -1,5 +1,5 @@
 ---
-description: "Use when performing an NNF software release, creating release branches, tagging releases, managing release PRs, running release-all.sh, comparing releases, or finalizing release notes. Covers releases across the 10 NNF repositories."
+description: "Use when performing an NNF software release, creating release branches, tagging releases, managing release PRs, running release-all.sh, comparing releases, or finalizing release notes. Covers releases across all NNF repositories."
 tools: [execute, read, edit, search, todo, web]
 argument-hint: "Specify the release type (patch, minor, or major) — defaults to patch"
 ---
diff --git a/tools/release-all/RELEASE-PLAN.md b/tools/release-all/RELEASE-PLAN.md
index 82d9e51..44639c3 100644
--- a/tools/release-all/RELEASE-PLAN.md
+++ b/tools/release-all/RELEASE-PLAN.md
@@ -13,6 +13,7 @@ This document is the authoritative, self-contained guide for executing an NNF so
 | `lustre_fs_operator` | `NearNodeFlash/lustre-fs-operator` | NearNodeFlash | `master` | |
 | `nnf_mfu` | `NearNodeFlash/nnf-mfu` | NearNodeFlash | `master` | Standalone, not a submodule |
 | `nnf_ec` | `NearNodeFlash/nnf-ec` | NearNodeFlash | `master` | Standalone, not a submodule |
+| `nnf_storedversions_maint` | `NearNodeFlash/nnf-storedversions-maint` | NearNodeFlash | **`main`** (not master) | Standalone, not a submodule |
 | `nnf_sos` | `NearNodeFlash/nnf-sos` | NearNodeFlash | `master` | Requires `-M` flag |
 | `nnf_dm` | `NearNodeFlash/nnf-dm` | NearNodeFlash | `master` | Vendors nnf-sos |
 | `nnf_integration_test` | `NearNodeFlash/nnf-integration-test` | NearNodeFlash | `master` | Vendors nnf-sos |
@@ -24,7 +25,7 @@ This document is the authoritative, self-contained guide for executing an NNF so
 **Dependency chain (release order):**
 
 ```text
-dws → lustre_csi_driver → lustre_fs_operator → nnf_mfu → nnf_ec → nnf_sos → nnf_dm → nnf_integration_test → nnf_deploy → nnf_doc
+dws → lustre_csi_driver → lustre_fs_operator → nnf_mfu → nnf_ec → nnf_storedversions_maint → nnf_sos → nnf_dm → nnf_integration_test → nnf_deploy → nnf_doc
 ```
 
 **Vendoring dependencies (peer modules in `go.mod`):**
@@ -33,6 +34,7 @@ dws → lustre_csi_driver → lustre_fs_operator → nnf_mfu → nnf_ec → nnf_
 - `nnf_sos` vendors `dws`, `lustre-fs-operator`, `nnf-ec`
 - `nnf_dm` vendors `dws`, `lustre-fs-operator`, `nnf-ec` (indirect), `nnf-sos`
 - `nnf_integration_test` vendors `dws`, `lustre-fs-operator`, `nnf-ec` (indirect), `nnf-sos`
+- `nnf_storedversions_maint` has **no** NNF peer vendoring dependencies
 
 If an upstream repo changes, **all downstream repos that vendor it** may need revendoring before the release (see Step 2a). The vendoring check in Step 2 catches these automatically.
 
@@ -108,7 +110,7 @@ List repos:
 ./release-all.sh -L
 ```
 
-Save the ordered list: `dws`, `lustre_csi_driver`, `lustre_fs_operator`, `nnf_mfu`, `nnf_ec`, `nnf_sos`, `nnf_dm`, `nnf_integration_test`, `nnf_deploy`, `nnf_doc`.
+Save the ordered list: `dws`, `lustre_csi_driver`, `lustre_fs_operator`, `nnf_mfu`, `nnf_ec`, `nnf_storedversions_maint`, `nnf_sos`, `nnf_dm`, `nnf_integration_test`, `nnf_deploy`, `nnf_doc`.
 
 #### Step 2 — Check vendoring *(sequential, per repo)*
 
@@ -159,7 +161,7 @@ done
 
 Review output for merge conflicts before continuing.
 
-> **Note:** `nnf_mfu` and `nnf_ec` are standalone repos that often have no new commits between releases. If either reports "No new changes to release", this is normal — **skip that repo in all subsequent phases** (Steps 4a–4d and the `nnf_doc` sequence). Do not attempt to push, PR, merge, or tag a repo that had no changes.
+> **Note:** `nnf_mfu`, `nnf_ec`, and `nnf_storedversions_maint` are standalone repos that often have no new commits between releases. If any reports "No new changes to release", this is normal — **skip that repo in all subsequent phases** (Steps 4a–4d and the `nnf_doc` sequence). Do not attempt to push, PR, merge, or tag a repo that had no changes.
 
 #### `nnf_doc` deferral
 
diff --git a/tools/release-all/final-release-notes.sh b/tools/release-all/final-release-notes.sh
index 5e5c4d9..a61a874 100755
--- a/tools/release-all/final-release-notes.sh
+++ b/tools/release-all/final-release-notes.sh
@@ -23,6 +23,7 @@ PROG=$(basename "$0")
 
 NNF_DOC_URL='git@github.com:NearNodeFlash/NearNodeFlash.github.io.git'
 NNF_DEPLOY_URL='git@github.com:NearNodeFlash/nnf-deploy.git'
+NNF_STOREDVERSIONS_MAINT_URL='git@github.com:NearNodeFlash/nnf-storedversions-maint.git'
 
 BOLD=$(tput bold)
 NORMAL=$(tput sgr0)
@@ -382,6 +383,11 @@ dep_url="$NNF_DEPLOY_URL"
 dep_name=$(get_repo_dir_name "$dep_url")
 check_release_vX "$dep_repo_short_name" "$dep_name" "$dep_url" "releases/v0"
 
+svm_repo_short_name="nnf_storedversions_maint"
+svm_url="$NNF_STOREDVERSIONS_MAINT_URL"
+svm_name=$(get_repo_dir_name "$svm_url")
+check_release_vX "$svm_repo_short_name" "$svm_name" "$svm_url" "releases/v0"
+
 doc_repo_short_name="nnf_doc"
 doc_url="$NNF_DOC_URL"
 doc_name=$(get_repo_dir_name "$doc_url")
diff --git a/tools/release-all/release-all.sh b/tools/release-all/release-all.sh
index 4a0ef8d..59e928e 100755
--- a/tools/release-all/release-all.sh
+++ b/tools/release-all/release-all.sh
@@ -27,22 +27,24 @@ repomap_keys[1]=lustre_csi_driver
 repomap_keys[2]=lustre_fs_operator
 repomap_keys[3]=nnf_mfu
 repomap_keys[4]=nnf_ec
-repomap_keys[5]=nnf_sos
-repomap_keys[6]=nnf_dm
-repomap_keys[7]=nnf_integration_test
-repomap_keys[8]=nnf_deploy
-repomap_keys[9]=nnf_doc
+repomap_keys[5]=nnf_storedversions_maint
+repomap_keys[6]=nnf_sos
+repomap_keys[7]=nnf_dm
+repomap_keys[8]=nnf_integration_test
+repomap_keys[9]=nnf_deploy
+repomap_keys[10]=nnf_doc
 
 declare "repomap_${repomap_keys[0]}"='git@github.com:DataWorkflowServices/dws.git'
 declare "repomap_${repomap_keys[1]}"='git@github.com:HewlettPackard/lustre-csi-driver.git'
 declare "repomap_${repomap_keys[2]}"='git@github.com:NearNodeFlash/lustre-fs-operator.git'
 declare "repomap_${repomap_keys[3]}"='git@github.com:NearNodeFlash/nnf-mfu.git'
 declare "repomap_${repomap_keys[4]}"='git@github.com:NearNodeFlash/nnf-ec.git'
-declare "repomap_${repomap_keys[5]}"='git@github.com:NearNodeFlash/nnf-sos.git'
-declare "repomap_${repomap_keys[6]}"='git@github.com:NearNodeFlash/nnf-dm.git'
-declare "repomap_${repomap_keys[7]}"='git@github.com:NearNodeFlash/nnf-integration-test.git'
-declare "repomap_${repomap_keys[8]}"='git@github.com:NearNodeFlash/nnf-deploy.git'
-declare "repomap_${repomap_keys[9]}"='git@github.com:NearNodeFlash/NearNodeFlash.github.io.git'
+declare "repomap_${repomap_keys[5]}"='git@github.com:NearNodeFlash/nnf-storedversions-maint.git'
+declare "repomap_${repomap_keys[6]}"='git@github.com:NearNodeFlash/nnf-sos.git'
+declare "repomap_${repomap_keys[7]}"='git@github.com:NearNodeFlash/nnf-dm.git'
+declare "repomap_${repomap_keys[8]}"='git@github.com:NearNodeFlash/nnf-integration-test.git'
+declare "repomap_${repomap_keys[9]}"='git@github.com:NearNodeFlash/nnf-deploy.git'
+declare "repomap_${repomap_keys[10]}"='git@github.com:NearNodeFlash/NearNodeFlash.github.io.git'
 
 getter() {
     # Getter for the associative-array-ish thingy that works in bash v3 for Mac.
@@ -235,6 +237,7 @@ get_default_branch() {
     local default_branch=master
 
     [[ $repo_short_name = nnf_doc ]] && default_branch=main
+    [[ $repo_short_name = nnf_storedversions_maint ]] && default_branch=main
     echo "$default_branch"
 }
 
@@ -512,6 +515,11 @@ update_own_release_references() {
         update_kustomization_file "$k_yaml" controller "$release_ver" "$indent"
         git add "$k_yaml"
         ;;
+    nnf_storedversions_maint)
+        k_yaml="config/manager"
+        update_kustomization_file "$k_yaml" controller "$release_ver" "$indent"
+        git add "$k_yaml"
+        ;;
     esac
 
     if [[ $(git status -s | wc -l) -gt 0 ]]; then
diff --git a/tools/release-all/setup-release-env.sh b/tools/release-all/setup-release-env.sh
index 8198f8c..2258b60 100755
--- a/tools/release-all/setup-release-env.sh
+++ b/tools/release-all/setup-release-env.sh
@@ -107,6 +107,7 @@ nnf_gh_repo() {
         lustre_fs_operator)   echo "NearNodeFlash/lustre-fs-operator" ;;
         nnf_mfu)              echo "NearNodeFlash/nnf-mfu" ;;
         nnf_ec)               echo "NearNodeFlash/nnf-ec" ;;
+        nnf_storedversions_maint) echo "NearNodeFlash/nnf-storedversions-maint" ;;
         nnf_sos)              echo "NearNodeFlash/nnf-sos" ;;
         nnf_dm)               echo "NearNodeFlash/nnf-dm" ;;
         nnf_integration_test) echo "NearNodeFlash/nnf-integration-test" ;;

From 871d3ce54ef72559b0591aee06e04506f97e689a Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Wed, 8 Apr 2026 09:51:01 -0500
Subject: [PATCH 6/9] Add the flux user to allow copy offload testing

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 tools/kind.sh | 77 ++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 16 deletions(-)

diff --git a/tools/kind.sh b/tools/kind.sh
index 078ac80..be4620d 100755
--- a/tools/kind.sh
+++ b/tools/kind.sh
@@ -22,6 +22,42 @@
 set -e
 set -o pipefail
 
+usage() {
+    echo "Usage: $0 [--no-argocd] <CMD>"
+    echo
+    echo "Options:"
+    echo "  --no-argocd             Deploy without ArgoCD using the legacy overlay."
+    echo "                          Installs cert-manager, mpi-operator, and lustre"
+    echo "                          operators directly instead of via ArgoCD."
+    echo
+    echo "Commands:"
+    echo "  create                  Create a new KIND cluster."
+    echo "  destroy                 Destroy the existing KIND cluster."
+    echo "  reset                   Destroy the existing KIND cluster and create"
+    echo "                          a new cluster. Note: locally-built images are"
+    echo "                          not automatically reloaded -- re-run 'push' or"
+    echo "                          './nnf-deploy make kind-push' after a reset."
+    echo "  push                    Execute 'make kind-push' in each submodule dir."
+    echo "  argocd_attach <new_password>"
+    echo "                          Login to the argocd instance and set the new"
+    echo "                          password. Then add the Git repo to the instance."
+    echo "  argocd_login <new_password>"
+    echo "                          Only do the login to the argocd instance and set"
+    echo "                          the password."
+    echo "  argocd_add_git_repo     Only add the git repo to the argocd instance."
+    echo "  help                    Show this help message."
+}
+
+NO_ARGOCD=
+while [[ "$1" == --* ]]; do
+    case "$1" in
+    --no-argocd) NO_ARGOCD=1 ;;
+    --help) usage; exit 0 ;;
+    *) echo "Unknown option: $1"; exit 1 ;;
+    esac
+    shift
+done
+
 CMD="$1"
 shift
 
@@ -77,6 +113,22 @@ function inject_ca_certs {
     [[ -n "$tmp_certs" ]] && rm -f "$tmp_certs"
 }
 
+# Create the integration test user on KIND Rabbit worker nodes.
+# The nnf-integration-test suite verifies that UID/GID exist on Rabbit nodes
+# (defaults: UID=1051 GID=1052, the flux user). Without this, the BeforeSuite
+# check fails and no tests run.
+function create_test_user {
+    local uid=${NNF_USER_ID:-1051}
+    local gid=${NNF_GROUP_ID:-1052}
+
+    echo "Creating test user (UID=$uid, GID=$gid) on KIND Rabbit worker nodes..."
+    for node in $(kind get nodes | grep -v control-plane); do
+        # Create group and user if they don't already exist
+        docker exec "$node" sh -c "getent group $gid >/dev/null 2>&1 || groupadd -g $gid testgroup"
+        docker exec "$node" sh -c "getent passwd $uid >/dev/null 2>&1 || useradd -u $uid -g $gid -M -s /bin/bash testuser"
+    done
+}
+
 function create_cluster {
     CONFIG=kind-config.yaml
 
@@ -146,7 +198,14 @@ EOF
     # Use the same init routines that we use on real hardware.
     # This applies taints and labels to rabbit nodes, and installs other
     # services that rabbit software requires.
+    if [[ -n $NO_ARGOCD ]]; then
+        echo "Deploying without ArgoCD (legacy overlay mode)..."
+        cp config/overlay-legacy.yaml-template overlay-legacy.yaml
+    fi
     ./nnf-deploy init
+
+    # Create the test user on Rabbit nodes for integration tests
+    create_test_user
 }
 
 function have_argocd {
@@ -231,23 +290,7 @@ function push_submodules {
     done
 }
 
-usage() {
-    echo "Usage: $0 <CMD>"
-    echo
-    echo "  create                  Create a new KIND cluster."
-    echo "  destroy                 Destroy the existing KIND cluster."
-    echo "  reset                   Destroy the existing KIND cluster and create"
-    echo "                          a new cluster."
-    echo "  push                    Execute 'make kind-push' in each submodule dir."
-    echo "  argocd_attach <new_password>"
-    echo "                          Login to the argocd instance and set the new"
-    echo "                          password. Then add the Git repo to the instance."
-    echo "  argocd_login <new_password>"
-    echo "                          Only do the login to the argocd instance and set"
-    echo "                          the password."
-    echo "  argocd_add_git_repo     Only add the git repo to the argocd instance."
 
-}
 
 if [[ "$CMD" == "create" ]]; then
     create_cluster
@@ -264,6 +307,8 @@ elif [[ "$CMD" == "argocd_login" ]]; then
     argocd_login "$*"
 elif [[ "$CMD" == "argocd_add_git_repo" ]]; then
     argocd_add_git_repo
+elif [[ "$CMD" == "help" || "$CMD" == "--help" ]]; then
+    usage
 else
     usage
     exit 1

From 1e2573a16027bacb241379318007cea7a03798ed Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Wed, 8 Apr 2026 15:57:18 -0500
Subject: [PATCH 7/9] Pull contributors for PRs from CONTRIBUTORS file

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 tools/release-all/setup-release-env.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/release-all/setup-release-env.sh b/tools/release-all/setup-release-env.sh
index 2258b60..b0cfe1f 100755
--- a/tools/release-all/setup-release-env.sh
+++ b/tools/release-all/setup-release-env.sh
@@ -64,10 +64,10 @@ if [[ ! -f "$CODEOWNERS_FILE" ]]; then
     echo "ERROR: $CODEOWNERS_FILE not found. Run from tools/release-all/ inside an nnf-deploy clone." >&2
     return 1
 fi
-REVIEWER_POOL=$(grep '^\*' "$CODEOWNERS_FILE" | tr -s ' ' | cut -d' ' -f2- | tr ' ' '\n' | sed 's/@//' | paste -sd,)
+REVIEWER_POOL=$(grep '^\*' "$CODEOWNERS_FILE" | tr -s ' ' | cut -d' ' -f2- | tr ' ' '\n' | sed 's/@//' | paste -sd, -)
 export RELEASE_REVIEWERS
 RELEASE_REVIEWERS=$(echo "$REVIEWER_POOL" | tr ',' '\n' \
-    | grep -v "$RELEASE_USER" | paste -sd,)
+    | grep -v "$RELEASE_USER" | paste -sd, -)
 
 if [[ -z "$RELEASE_REVIEWERS" ]]; then
     echo "WARNING: RELEASE_REVIEWERS is empty — you are the only code owner in CODEOWNERS." >&2

From a788abd584debcbcdc246fb549f31ad8ea55b092 Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Thu, 9 Apr 2026 11:44:32 -0500
Subject: [PATCH 8/9] Fix nnf_cmd exit code and update submodule pointers

Preserve the exit code of release-all.sh through the pipe in nnf_cmd
by returning PIPESTATUS[0]. Previously, all invocations exited 0 because
the pipe's exit code was that of 'cat', not release-all.sh.

Also advance submodule pointers to their current master HEADs:
- lustre-fs-operator: 7ebec7d -> c59379b (PR #125 update-vendor)
- nnf-dm:             42a3d3a4 -> c959e3b5 (PR #360 update-vendor + others)
- nnf-integration-test: b967cc0 -> 3e52df1 (PR #160 update-vendor + #159)
- nnf-sos:            8749851d -> bcb00b92 (PR #604 update-vendor + others)

Also restore the ONE-command-per-turn rule to release.agent.md and
remove trailing blank lines from RELEASE-PLAN.md.

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 .github/agents/release.agent.md        | 1 +
 lustre-fs-operator                     | 2 +-
 nnf-dm                                 | 2 +-
 nnf-integration-test                   | 2 +-
 nnf-sos                                | 2 +-
 tools/release-all/RELEASE-PLAN.md      | 4 ++--
 tools/release-all/setup-release-env.sh | 5 ++++-
 7 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/.github/agents/release.agent.md b/.github/agents/release.agent.md
index b60af6a..8c3ab7b 100644
--- a/.github/agents/release.agent.md
+++ b/.github/agents/release.agent.md
@@ -18,6 +18,7 @@ Before starting any release work, read these files completely:
 ## Core Rules
 
 - **ALWAYS wait for explicit user approval before proceeding to the next step.** After every step, state what you did, whether it succeeded, and the evidence. **WAIT for the user.**
+- **ONE command per turn.** Run the command for ONE repo, show the full output, state PASS or FAIL with evidence, then STOP. Do not run the next repo's command until the user explicitly says to proceed. This applies even when the plan shows a `for` loop — execute the loop body manually, one repo per turn.
 - **ALWAYS run commands in a terminal** so the user can see them.
 - **ALWAYS append `2>&1 | cat`** to every `release-all.sh` and `gh` command — no exceptions.
 - **ONE repo at a time.** Execute one phase per repo sequentially. **NEVER parallelize** across repos.
diff --git a/lustre-fs-operator b/lustre-fs-operator
index 7ebec7d..c59379b 160000
--- a/lustre-fs-operator
+++ b/lustre-fs-operator
@@ -1 +1 @@
-Subproject commit 7ebec7d6a4db230f53136c0a90d2cce46568eb2e
+Subproject commit c59379bba6df432d8a6f63c802d310c897058bbc
diff --git a/nnf-dm b/nnf-dm
index 42a3d3a..c959e3b 160000
--- a/nnf-dm
+++ b/nnf-dm
@@ -1 +1 @@
-Subproject commit 42a3d3a4efd0e8389e5104d19ad2c668de1baff9
+Subproject commit c959e3b573fe13136d55c734cedd41ac637da1a1
diff --git a/nnf-integration-test b/nnf-integration-test
index b967cc0..3e52df1 160000
--- a/nnf-integration-test
+++ b/nnf-integration-test
@@ -1 +1 @@
-Subproject commit b967cc079bb702f75661a228b52f28feb856603d
+Subproject commit 3e52df1dcef63e1954d6ca26c6b6895a6cc048e9
diff --git a/nnf-sos b/nnf-sos
index 8749851..bcb00b9 160000
--- a/nnf-sos
+++ b/nnf-sos
@@ -1 +1 @@
-Subproject commit 8749851dbc78c934d5f59302b3837a114a778fdd
+Subproject commit bcb00b928378bc0eea29be6590ae7adfb09578d3
diff --git a/tools/release-all/RELEASE-PLAN.md b/tools/release-all/RELEASE-PLAN.md
index 44639c3..66af28c 100644
--- a/tools/release-all/RELEASE-PLAN.md
+++ b/tools/release-all/RELEASE-PLAN.md
@@ -102,6 +102,8 @@ Source the release environment helper to auto-compute versions, detect the relea
 source ./setup-release-env.sh -B patch   # or -B minor / -B major
 ```
 
+> **Important:** Never pipe this `source` command (e.g. `source ... | cat`). Piping runs `source` in a subshell, which means the exported variables and function definitions (`nnf_cmd`, etc.) are lost to the calling shell. If you need to capture output, redirect to a file or run without the pipe.
+
 This exports `$PREVIOUS_RELEASE`, `$NNF_RELEASE`, `$RELEASE_TYPE`, and `$RELEASE_REVIEWERS`, and defines helper functions `nnf_cmd`, `nnf_create_pr`, `nnf_add_reviewers`, and `nnf_gh_repo`. Confirm the printed summary with the user before proceeding.
 
 List repos:
@@ -270,8 +272,6 @@ The `-i` flag displays image version changes inline. Use `-d` to display the ful
 
 Check each repo for correct tag, release notes, and artifacts (`manifests.tar` + `manifests-kind.tar` on `nnf-deploy`).
 
-
-
 ---
 
 ### Relevant files
diff --git a/tools/release-all/setup-release-env.sh b/tools/release-all/setup-release-env.sh
index b0cfe1f..e1ff34f 100755
--- a/tools/release-all/setup-release-env.sh
+++ b/tools/release-all/setup-release-env.sh
@@ -78,9 +78,12 @@ fi
 # Usage: nnf_cmd <phase> <repo>
 nnf_cmd() {
     local phase="$1" repo="$2"
-    local args=()
+    local args=() _rc
     [[ "$repo" == "nnf_sos" ]] && args+=("-M")
     ./release-all.sh -B "$RELEASE_TYPE" -P "$phase" -R "$repo" "${args[@]}" 2>&1 | cat
+    # bash uses PIPESTATUS[0]; zsh uses pipestatus[1] (lowercase, 1-indexed)
+    _rc=${PIPESTATUS[0]:-${pipestatus[1]}}
+    return "${_rc:-0}"
 }
 
 # --- Helper: create-pr + capture PR number -----------------------------------

From ac7e6ef8421f635de62a77d464829ff45529c61c Mon Sep 17 00:00:00 2001
From: Anthony Floeder <anthony.floeder@hpe.com>
Date: Thu, 9 Apr 2026 13:43:37 -0500
Subject: [PATCH 9/9] Update lustre-fs-operator or lustre-csi-driver release
 references

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
---
 config/repositories.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/repositories.yaml b/config/repositories.yaml
index 641ae8f..cd59bc7 100644
--- a/config/repositories.yaml
+++ b/config/repositories.yaml
@@ -41,7 +41,7 @@ repositories:
     master: https://ghcr.io/nearnodeflash/lustre-fs-operator
     useRemoteK: false
     remoteReference:
-      build: v0.1.17
+      build: v0.1.18
       url: https://github.com/NearNodeFlash/lustre-fs-operator.git/config/default/?ref=%s
 buildConfiguration:
   # Environment variables to set when calling any 'make' or 'deploy'